fix: hold the deploy lock across git-from-directory and harden the advisory lock

git-from-directory updated the deploy branch before taking the deploy
lock and rolled it back after releasing it. The lock is now held for the
whole update, build, and rollback, and the rollback only moves the branch
if it still points at the commit that was created.

A failed lock attempt and apps:lock both truncated the lock file,
dropping the build id of the in-progress deploy that builds:cancel and
builds:output read. Releasing the lock unlocked before removing the
file, so a waiter could end up holding a lock on a deleted file while
another deploy locked a new one. The file is now removed before
unlocking, and acquiring retries when the locked file is no longer the
one at the lock path.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-10-11 16:21:13 -04:00
1 parent 7955e8ee06
commit 11c4e908d5
8 files changed
+312 -56

No files matched your search

+71 -9
View File
@@ -51,12 +51,16 @@ deploy_second_procfile_image() {
hold_deploy_lock() {
DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
sudo -u dokku flock -o "$DEPLOY_LOCK_FILE" sleep 120 >/dev/null 2>&1 3>&- &
DEPLOY_LOCK_HOLDER_PID="$!"
DEPLOY_LOCK_HOLDER_PID_FILE="$(mktemp "/tmp/${DOKKU_DOMAIN}.XXXXX")"
chown dokku:dokku "$DEPLOY_LOCK_HOLDER_PID_FILE"
# the holder writes a build id like a deploy would, then becomes the sleep
# process so that killing its pid releases the lock
sudo -u dokku bash -c "exec 200>>'$DEPLOY_LOCK_FILE'; flock -n 200 || exit 1; echo test-lock-holder >'$DEPLOY_LOCK_FILE'; echo \$\$ >'$DEPLOY_LOCK_HOLDER_PID_FILE'; exec sleep 120" >/dev/null 2>&1 3>&- &
local i
for i in $(seq 1 20); do
if ! flock -n "$DEPLOY_LOCK_FILE" true; then
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]] && ! flock -n "$DEPLOY_LOCK_FILE" true; then
return 0
fi
sleep 0.5
@@ -67,15 +71,25 @@ hold_deploy_lock() {
}
release_deploy_lock() {
if [[ -z "$DEPLOY_LOCK_HOLDER_PID" ]]; then
if [[ -z "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
return 0
fi
kill "$DEPLOY_LOCK_HOLDER_PID" &>/dev/null || true
wait "$DEPLOY_LOCK_HOLDER_PID" &>/dev/null || true
pkill -u dokku -f "flock -o $DEPLOY_LOCK_FILE" &>/dev/null || true
rm -f "$DEPLOY_LOCK_FILE"
DEPLOY_LOCK_HOLDER_PID=""
local holder_pid i
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
holder_pid="$(cat "$DEPLOY_LOCK_HOLDER_PID_FILE")"
kill "$holder_pid" &>/dev/null || true
for i in $(seq 1 20); do
kill -0 "$holder_pid" &>/dev/null || break
sleep 0.5
done
fi
rm -f "$DEPLOY_LOCK_FILE" "$DEPLOY_LOCK_HOLDER_PID_FILE"
DEPLOY_LOCK_HOLDER_PID_FILE=""
}
deploy_branch_rev() {
sudo -u dokku git -C "/home/dokku/$TEST_APP" rev-parse master
}
@test "(git) git:from-image [missing]" {
@@ -113,11 +127,21 @@ release_deploy_lock() {
assert_output_contains "Image exists on host, skipping pull"
assert_output_contains "No changes detected, skipping git commit"
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku git:from-image --force $TEST_APP linuxserver/foldingathome:7.6.21"
echo "output: $output"
echo "status: $status"
assert_output_contains "Force pulling image"
assert_output_contains "No changes detected, skipping git commit"
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
}
@test "(git) git:from-image [normal-custom-branch]" {
@@ -210,6 +234,11 @@ release_deploy_lock() {
assert_success
assert_output ""
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
echo "output: $output"
echo "status: $status"
@@ -236,6 +265,11 @@ release_deploy_lock() {
echo "status: $status"
assert_failure
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
echo "output: $output"
echo "status: $status"
@@ -500,6 +534,10 @@ EOF
echo "status: $status"
assert_success
local DEPLOY_REV
DEPLOY_REV="$(deploy_branch_rev)"
echo "deploy branch rev: $DEPLOY_REV"
hold_deploy_lock
run /bin/bash -c "dokku git:from-image $TEST_APP dokku-test/$TEST_APP:default-procfile"
@@ -514,6 +552,18 @@ EOF
assert_success
assert_output "dokku-test/$TEST_APP:second-procfile"
run deploy_branch_rev
echo "output: $output"
echo "status: $status"
assert_success
assert_output "$DEPLOY_REV"
run /bin/bash -c "cat $DEPLOY_LOCK_FILE"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "test-lock-holder"
run /bin/bash -c "dokku git:sync $TEST_APP https://github.com/dokku/smoke-test-app.git 2.0.0"
echo "output: $output"
echo "status: $status"
@@ -526,6 +576,18 @@ EOF
assert_success
assert_output "dokku-test/$TEST_APP:second-procfile"
run deploy_branch_rev
echo "output: $output"
echo "status: $status"
assert_success
assert_output "$DEPLOY_REV"
run /bin/bash -c "cat $DEPLOY_LOCK_FILE"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "test-lock-holder"
release_deploy_lock
run /bin/bash -c "dokku git:from-image $TEST_APP dokku-test/$TEST_APP:default-procfile"