fix: release deploy locks on exit and honor apps:lock

A deploy that exited before releasing its lock, such as after a failed build, left `.deploy.lock` behind, so `apps:locked` reported a lock nothing held and the build record stayed running. The lock is now released and the record finalized whenever the deploy exits. `apps:lock` never actually blocked deploys, as it only created the file deploys lock. It now creates its own lock that git pushes, `ps:rebuild`, `git:sync`, and the `git:from-*` commands refuse to deploy past, and `apps:locked` only reports a lock when the app was locked via `apps:lock` or a deploy is in progress. Locks created by `apps:lock` in older versions are migrated on install.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-10-24 13:58:47 -04:00
1 parent 57ae817dab
commit c4b164a270
17 files changed
+535 -69

No files matched your search

+3
View File
@@ -0,0 +1,3 @@
FROM alpine:3.20
RUN false
+128 -1
View File
@@ -9,6 +9,7 @@ setup() {
}
teardown() {
release_deploy_lock
rm -rf "${PLUGIN_ENABLED_PATH:?}/$APP_FILTER_PLUGIN_NAME" "${PLUGIN_AVAILABLE_PATH:?}/$APP_FILTER_PLUGIN_NAME"
dokku --force apps:destroy "$TEST_APP-source" &>/dev/null || true
global_teardown
@@ -402,7 +403,7 @@ teardown() {
destroy_app
}
@test "(apps) apps:lock does not truncate an existing lock file" {
@test "(apps) apps:lock does not modify the deploy lock file" {
run create_app
echo "output: $output"
echo "status: $status"
@@ -432,6 +433,132 @@ teardown() {
destroy_app
}
@test "(apps:locked) ignores a deploy lock file that no deploy holds" {
run create_app
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "echo stale-id >$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock && chown dokku:dokku $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku apps:report $TEST_APP --app-locked"
echo "output: $output"
echo "status: $status"
assert_output "false"
destroy_app
}
@test "(apps:locked) reports a deploy lock held by a running deploy" {
run create_app
echo "output: $output"
echo "status: $status"
assert_success
hold_deploy_lock
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:unlock $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "A deploy may be in progress."
release_deploy_lock
destroy_app
}
@test "(apps:unlock) succeeds for an app that is not locked" {
run create_app
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:unlock $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_not_contains "A deploy may be in progress."
destroy_app
}
@test "(apps:rename) fails for a locked app" {
run create_app
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:lock $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:rename $TEST_APP great-test-name"
echo "output: $output"
echo "status: $status"
assert_failure
assert_output_contains "Unable to rename a locked app"
run /bin/bash -c "dokku apps:exists great-test-name"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
destroy_app
}
@test "(apps) [install] converts a lock created by apps:lock in older versions" {
run create_app
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "sudo -u dokku touch $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "DOKKU_ROOT=$DOKKU_ROOT PLUGIN_PATH=$PLUGIN_PATH PLUGIN_AVAILABLE_PATH=$PLUGIN_AVAILABLE_PATH PLUGIN_ENABLED_PATH=$PLUGIN_ENABLED_PATH PLUGIN_CORE_AVAILABLE_PATH=$PLUGIN_CORE_AVAILABLE_PATH DOKKU_LIB_ROOT=$DOKKU_LIB_ROOT $PLUGIN_ENABLED_PATH/apps/install"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "[[ ! -e $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock ]]"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "[[ -f $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.manual-deploy.lock ]]"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
destroy_app
}
@test "(apps:report) emits new stripped JSON keys alongside legacy" {
create_app
+32
View File
@@ -33,6 +33,38 @@ teardown() {
assert_success
}
@test "(apps:clone) does not copy the lock of a locked app" {
run /bin/bash -c "dokku apps:create $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:lock $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:clone --skip-deploy $TEST_APP great-test-name"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:locked great-test-name"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku --force apps:destroy great-test-name"
echo "output: $output"
echo "status: $status"
assert_success
}
@test "(apps) apps:clone (no app)" {
run /bin/bash -c "dokku apps:create $TEST_APP"
echo "output: $output"
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env bats
load test_helper
setup() {
global_setup
create_app
}
teardown() {
destroy_app
global_teardown
}
@test "(apps:lock) blocks ps:rebuild until apps:unlock" {
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku builds:list $TEST_APP --format json | jq length"
echo "output: $output"
echo "status: $status"
assert_success
local BUILD_COUNT="$output"
run /bin/bash -c "dokku apps:lock $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
assert_output_contains "$TEST_APP has been locked via apps:lock" -1
assert_output_not_contains "Building $TEST_APP"
run /bin/bash -c "dokku builds:list $TEST_APP --format json | jq length"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "$BUILD_COUNT"
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:unlock $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
}
@test "(apps:lock) blocks git:sync before fetching" {
run /bin/bash -c "dokku apps:lock $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git"
echo "output: $output"
echo "status: $status"
assert_failure
assert_output_contains "$TEST_APP has been locked via apps:lock"
run /bin/bash -c "sudo -u dokku git -C $DOKKU_ROOT/$TEST_APP rev-parse --verify -q master"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku builds:list $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "No builds recorded"
}
+69
View File
@@ -101,6 +101,75 @@ EOF
assert_output_contains "abandoned"
}
@test "(builds) [lock] a failed build releases the deploy lock and finalizes the record as failed" {
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku builder-dockerfile:set $TEST_APP dockerfile-path fail.Dockerfile"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
assert_output_contains "App build failed" -1
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "[[ ! -e $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock ]]"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku builds:report $TEST_APP --build-status"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "failed"
run /bin/bash -c "dokku builds:report $TEST_APP --build-exit-code"
echo "output: $output"
echo "status: $status"
assert_success
assert_not_output "0"
}
@test "(builds) [lock] a deploy that exits while holding the lock finalizes its record" {
run /bin/bash -c "sudo -u dokku env PLUGIN_PATH=$PLUGIN_PATH PLUGIN_AVAILABLE_PATH=$PLUGIN_AVAILABLE_PATH PLUGIN_ENABLED_PATH=$PLUGIN_ENABLED_PATH PLUGIN_CORE_AVAILABLE_PATH=$PLUGIN_CORE_AVAILABLE_PATH DOKKU_LIB_ROOT=$DOKKU_LIB_ROOT DOKKU_BUILD_ID=exit0001 bash -c 'source $PLUGIN_CORE_AVAILABLE_PATH/common/functions; plugn trigger builds-record-start $TEST_APP exit0001 \$\$ git-hook; acquire_app_deploy_lock $TEST_APP exclusive; exit 3'"
echo "output: $output"
echo "status: $status"
assert_equal 3 "$status"
run /bin/bash -c "[[ ! -e $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock ]]"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku apps:locked $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "dokku builds:info $TEST_APP exit0001 --format json | jq -r .status"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "failed"
run /bin/bash -c "dokku builds:info $TEST_APP exit0001 --format json | jq -r .exit_code"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "3"
}
@test "(builds:list --kind=build) filters to build-kind records" {
write_finished_record "$TEST_APP" "b1" "succeeded" "git-hook" 1001 "build"
write_finished_record "$TEST_APP" "d1" "succeeded" "ps:restart" 1002 "deploy"
+24
View File
@@ -90,6 +90,30 @@ wait_for() {
assert_output_contains "lock held"
}
@test "(deploy-lock) the lock file is removed when the holder exits without releasing it" {
run /bin/bash -c "$(lock_script "holder-a" "acquire_advisory_lock '$LOCK_FILE' exclusive 'waiting' 'lock held'; exit 1")"
echo "output: $output"
echo "status: $status"
assert_failure
run /bin/bash -c "[[ ! -e '$LOCK_FILE' ]]"
echo "output: $output"
echo "status: $status"
assert_success
}
@test "(deploy-lock) the lock file is removed when the holder is terminated" {
start_lock_holder "holder-a"
kill -TERM "${LOCK_PIDS[0]}"
wait_for "! kill -0 ${LOCK_PIDS[0]} 2>/dev/null"
run /bin/bash -c "[[ ! -e '$LOCK_FILE' ]]"
echo "output: $output"
echo "status: $status"
assert_success
}
@test "(deploy-lock) release only removes the lock file it holds" {
start_lock_holder "holder-a"
-39
View File
@@ -49,45 +49,6 @@ deploy_second_procfile_image() {
assert_output_contains "second.Procfile"
}
hold_deploy_lock() {
DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
DEPLOY_LOCK_HOLDER_PID_FILE="$(mktemp "/tmp/${DOKKU_DOMAIN}.XXXXX")"
chown dokku:dokku "$DEPLOY_LOCK_HOLDER_PID_FILE"
# the holder writes a build id like a deploy would, then becomes the sleep
# process so that killing its pid releases the lock
sudo -u dokku bash -c "exec 200>>'$DEPLOY_LOCK_FILE'; flock -n 200 || exit 1; echo test-lock-holder >'$DEPLOY_LOCK_FILE'; echo \$\$ >'$DEPLOY_LOCK_HOLDER_PID_FILE'; exec sleep 120" >/dev/null 2>&1 3>&- &
local i
for i in $(seq 1 20); do
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]] && ! flock -n "$DEPLOY_LOCK_FILE" true; then
return 0
fi
sleep 0.5
done
echo "unable to acquire the deploy lock for $TEST_APP"
return 1
}
release_deploy_lock() {
if [[ -z "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
return 0
fi
local holder_pid i
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
holder_pid="$(cat "$DEPLOY_LOCK_HOLDER_PID_FILE")"
kill "$holder_pid" &>/dev/null || true
for i in $(seq 1 20); do
kill -0 "$holder_pid" &>/dev/null || break
sleep 0.5
done
fi
rm -f "$DEPLOY_LOCK_FILE" "$DEPLOY_LOCK_HOLDER_PID_FILE"
DEPLOY_LOCK_HOLDER_PID_FILE=""
}
deploy_branch_rev() {
sudo -u dokku git -C "/home/dokku/$TEST_APP" rev-parse master
}
+41
View File
@@ -251,6 +251,47 @@ run_plugin_script() {
run /bin/bash -c "PLUGIN_PATH=$PLUGIN_PATH PLUGIN_AVAILABLE_PATH=$PLUGIN_AVAILABLE_PATH PLUGIN_ENABLED_PATH=$PLUGIN_ENABLED_PATH PLUGIN_CORE_AVAILABLE_PATH=$PLUGIN_CORE_AVAILABLE_PATH DOKKU_LIB_ROOT=$DOKKU_LIB_ROOT $PLUGIN_ENABLED_PATH/$PLUGIN_NAME/$SCRIPT_NAME $*"
}
# Hold the deploy lock for $TEST_APP from a background process, as an in-progress deploy would.
# Call release_deploy_lock (typically from teardown) to release it.
hold_deploy_lock() {
DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
DEPLOY_LOCK_HOLDER_PID_FILE="$(mktemp "/tmp/${DOKKU_DOMAIN}.XXXXX")"
chown dokku:dokku "$DEPLOY_LOCK_HOLDER_PID_FILE"
# the holder writes a build id like a deploy would, then becomes the sleep
# process so that killing its pid releases the lock
sudo -u dokku bash -c "exec 200>>'$DEPLOY_LOCK_FILE'; flock -n 200 || exit 1; echo test-lock-holder >'$DEPLOY_LOCK_FILE'; echo \$\$ >'$DEPLOY_LOCK_HOLDER_PID_FILE'; exec sleep 120" >/dev/null 2>&1 3>&- &
local i
for i in $(seq 1 20); do
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]] && ! flock -n "$DEPLOY_LOCK_FILE" true; then
return 0
fi
sleep 0.5
done
echo "unable to acquire the deploy lock for $TEST_APP"
return 1
}
release_deploy_lock() {
if [[ -z "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
return 0
fi
local holder_pid i
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
holder_pid="$(cat "$DEPLOY_LOCK_HOLDER_PID_FILE")"
kill "$holder_pid" &>/dev/null || true
for i in $(seq 1 20); do
kill -0 "$holder_pid" &>/dev/null || break
sleep 0.5
done
fi
rm -f "$DEPLOY_LOCK_FILE" "$DEPLOY_LOCK_HOLDER_PID_FILE"
DEPLOY_LOCK_HOLDER_PID_FILE=""
}
check_urls() {
local PATTERN="$1"
run /bin/bash -c "dokku urls $TEST_APP | grep -E \"${PATTERN}\""