fix: release deploy locks on exit and honor apps:lock
A deploy that exited before releasing its lock, such as after a failed build, left `.deploy.lock` behind, so `apps:locked` reported a lock nothing held and the build record stayed running. The lock is now released and the record finalized whenever the deploy exits. `apps:lock` never actually blocked deploys, as it only created the file deploys lock. It now creates its own lock that git pushes, `ps:rebuild`, `git:sync`, and the `git:from-*` commands refuse to deploy past, and `apps:locked` only reports a lock when the app was locked via `apps:lock` or a deploy is in progress. Locks created by `apps:lock` in older versions are migrated on install.
This commit is contained in:
17 files changed
+535
-69
No files matched your search
@@ -0,0 +1,3 @@
|
||||
FROM alpine:3.20
|
||||
|
||||
RUN false
|
||||
+128
-1
@@ -9,6 +9,7 @@ setup() {
|
||||
}
|
||||
|
||||
teardown() {
|
||||
release_deploy_lock
|
||||
rm -rf "${PLUGIN_ENABLED_PATH:?}/$APP_FILTER_PLUGIN_NAME" "${PLUGIN_AVAILABLE_PATH:?}/$APP_FILTER_PLUGIN_NAME"
|
||||
dokku --force apps:destroy "$TEST_APP-source" &>/dev/null || true
|
||||
global_teardown
|
||||
@@ -402,7 +403,7 @@ teardown() {
|
||||
destroy_app
|
||||
}
|
||||
|
||||
@test "(apps) apps:lock does not truncate an existing lock file" {
|
||||
@test "(apps) apps:lock does not modify the deploy lock file" {
|
||||
run create_app
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
@@ -432,6 +433,132 @@ teardown() {
|
||||
destroy_app
|
||||
}
|
||||
|
||||
@test "(apps:locked) ignores a deploy lock file that no deploy holds" {
|
||||
run create_app
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "echo stale-id >$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock && chown dokku:dokku $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "dokku apps:report $TEST_APP --app-locked"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_output "false"
|
||||
|
||||
destroy_app
|
||||
}
|
||||
|
||||
@test "(apps:locked) reports a deploy lock held by a running deploy" {
|
||||
run create_app
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
hold_deploy_lock
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:unlock $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output_contains "A deploy may be in progress."
|
||||
|
||||
release_deploy_lock
|
||||
destroy_app
|
||||
}
|
||||
|
||||
@test "(apps:unlock) succeeds for an app that is not locked" {
|
||||
run create_app
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:unlock $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output_not_contains "A deploy may be in progress."
|
||||
|
||||
destroy_app
|
||||
}
|
||||
|
||||
@test "(apps:rename) fails for a locked app" {
|
||||
run create_app
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:lock $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:rename $TEST_APP great-test-name"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
assert_output_contains "Unable to rename a locked app"
|
||||
|
||||
run /bin/bash -c "dokku apps:exists great-test-name"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
destroy_app
|
||||
}
|
||||
|
||||
@test "(apps) [install] converts a lock created by apps:lock in older versions" {
|
||||
run create_app
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "sudo -u dokku touch $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "DOKKU_ROOT=$DOKKU_ROOT PLUGIN_PATH=$PLUGIN_PATH PLUGIN_AVAILABLE_PATH=$PLUGIN_AVAILABLE_PATH PLUGIN_ENABLED_PATH=$PLUGIN_ENABLED_PATH PLUGIN_CORE_AVAILABLE_PATH=$PLUGIN_CORE_AVAILABLE_PATH DOKKU_LIB_ROOT=$DOKKU_LIB_ROOT $PLUGIN_ENABLED_PATH/apps/install"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "[[ ! -e $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock ]]"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "[[ -f $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.manual-deploy.lock ]]"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
destroy_app
|
||||
}
|
||||
|
||||
@test "(apps:report) emits new stripped JSON keys alongside legacy" {
|
||||
create_app
|
||||
|
||||
|
||||
@@ -33,6 +33,38 @@ teardown() {
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "(apps:clone) does not copy the lock of a locked app" {
|
||||
run /bin/bash -c "dokku apps:create $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:lock $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:clone --skip-deploy $TEST_APP great-test-name"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:locked great-test-name"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku --force apps:destroy great-test-name"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "(apps) apps:clone (no app)" {
|
||||
run /bin/bash -c "dokku apps:create $TEST_APP"
|
||||
echo "output: $output"
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
load test_helper
|
||||
|
||||
setup() {
|
||||
global_setup
|
||||
create_app
|
||||
}
|
||||
|
||||
teardown() {
|
||||
destroy_app
|
||||
global_teardown
|
||||
}
|
||||
|
||||
@test "(apps:lock) blocks ps:rebuild until apps:unlock" {
|
||||
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku builds:list $TEST_APP --format json | jq length"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
local BUILD_COUNT="$output"
|
||||
|
||||
run /bin/bash -c "dokku apps:lock $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
assert_output_contains "$TEST_APP has been locked via apps:lock" -1
|
||||
assert_output_not_contains "Building $TEST_APP"
|
||||
|
||||
run /bin/bash -c "dokku builds:list $TEST_APP --format json | jq length"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "$BUILD_COUNT"
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:unlock $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "(apps:lock) blocks git:sync before fetching" {
|
||||
run /bin/bash -c "dokku apps:lock $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
assert_output_contains "$TEST_APP has been locked via apps:lock"
|
||||
|
||||
run /bin/bash -c "sudo -u dokku git -C $DOKKU_ROOT/$TEST_APP rev-parse --verify -q master"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "dokku builds:list $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output_contains "No builds recorded"
|
||||
}
|
||||
@@ -101,6 +101,75 @@ EOF
|
||||
assert_output_contains "abandoned"
|
||||
}
|
||||
|
||||
@test "(builds) [lock] a failed build releases the deploy lock and finalizes the record as failed" {
|
||||
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku builder-dockerfile:set $TEST_APP dockerfile-path fail.Dockerfile"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
assert_output_contains "App build failed" -1
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "[[ ! -e $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock ]]"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku builds:report $TEST_APP --build-status"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "failed"
|
||||
|
||||
run /bin/bash -c "dokku builds:report $TEST_APP --build-exit-code"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_not_output "0"
|
||||
}
|
||||
|
||||
@test "(builds) [lock] a deploy that exits while holding the lock finalizes its record" {
|
||||
run /bin/bash -c "sudo -u dokku env PLUGIN_PATH=$PLUGIN_PATH PLUGIN_AVAILABLE_PATH=$PLUGIN_AVAILABLE_PATH PLUGIN_ENABLED_PATH=$PLUGIN_ENABLED_PATH PLUGIN_CORE_AVAILABLE_PATH=$PLUGIN_CORE_AVAILABLE_PATH DOKKU_LIB_ROOT=$DOKKU_LIB_ROOT DOKKU_BUILD_ID=exit0001 bash -c 'source $PLUGIN_CORE_AVAILABLE_PATH/common/functions; plugn trigger builds-record-start $TEST_APP exit0001 \$\$ git-hook; acquire_app_deploy_lock $TEST_APP exclusive; exit 3'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_equal 3 "$status"
|
||||
|
||||
run /bin/bash -c "[[ ! -e $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock ]]"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku apps:locked $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "dokku builds:info $TEST_APP exit0001 --format json | jq -r .status"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "failed"
|
||||
|
||||
run /bin/bash -c "dokku builds:info $TEST_APP exit0001 --format json | jq -r .exit_code"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "3"
|
||||
}
|
||||
|
||||
@test "(builds:list --kind=build) filters to build-kind records" {
|
||||
write_finished_record "$TEST_APP" "b1" "succeeded" "git-hook" 1001 "build"
|
||||
write_finished_record "$TEST_APP" "d1" "succeeded" "ps:restart" 1002 "deploy"
|
||||
|
||||
@@ -90,6 +90,30 @@ wait_for() {
|
||||
assert_output_contains "lock held"
|
||||
}
|
||||
|
||||
@test "(deploy-lock) the lock file is removed when the holder exits without releasing it" {
|
||||
run /bin/bash -c "$(lock_script "holder-a" "acquire_advisory_lock '$LOCK_FILE' exclusive 'waiting' 'lock held'; exit 1")"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "[[ ! -e '$LOCK_FILE' ]]"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "(deploy-lock) the lock file is removed when the holder is terminated" {
|
||||
start_lock_holder "holder-a"
|
||||
|
||||
kill -TERM "${LOCK_PIDS[0]}"
|
||||
wait_for "! kill -0 ${LOCK_PIDS[0]} 2>/dev/null"
|
||||
|
||||
run /bin/bash -c "[[ ! -e '$LOCK_FILE' ]]"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "(deploy-lock) release only removes the lock file it holds" {
|
||||
start_lock_holder "holder-a"
|
||||
|
||||
|
||||
@@ -49,45 +49,6 @@ deploy_second_procfile_image() {
|
||||
assert_output_contains "second.Procfile"
|
||||
}
|
||||
|
||||
hold_deploy_lock() {
|
||||
DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
|
||||
DEPLOY_LOCK_HOLDER_PID_FILE="$(mktemp "/tmp/${DOKKU_DOMAIN}.XXXXX")"
|
||||
chown dokku:dokku "$DEPLOY_LOCK_HOLDER_PID_FILE"
|
||||
|
||||
# the holder writes a build id like a deploy would, then becomes the sleep
|
||||
# process so that killing its pid releases the lock
|
||||
sudo -u dokku bash -c "exec 200>>'$DEPLOY_LOCK_FILE'; flock -n 200 || exit 1; echo test-lock-holder >'$DEPLOY_LOCK_FILE'; echo \$\$ >'$DEPLOY_LOCK_HOLDER_PID_FILE'; exec sleep 120" >/dev/null 2>&1 3>&- &
|
||||
|
||||
local i
|
||||
for i in $(seq 1 20); do
|
||||
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]] && ! flock -n "$DEPLOY_LOCK_FILE" true; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "unable to acquire the deploy lock for $TEST_APP"
|
||||
return 1
|
||||
}
|
||||
|
||||
release_deploy_lock() {
|
||||
if [[ -z "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local holder_pid i
|
||||
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
|
||||
holder_pid="$(cat "$DEPLOY_LOCK_HOLDER_PID_FILE")"
|
||||
kill "$holder_pid" &>/dev/null || true
|
||||
for i in $(seq 1 20); do
|
||||
kill -0 "$holder_pid" &>/dev/null || break
|
||||
sleep 0.5
|
||||
done
|
||||
fi
|
||||
rm -f "$DEPLOY_LOCK_FILE" "$DEPLOY_LOCK_HOLDER_PID_FILE"
|
||||
DEPLOY_LOCK_HOLDER_PID_FILE=""
|
||||
}
|
||||
|
||||
deploy_branch_rev() {
|
||||
sudo -u dokku git -C "/home/dokku/$TEST_APP" rev-parse master
|
||||
}
|
||||
|
||||
@@ -251,6 +251,47 @@ run_plugin_script() {
|
||||
run /bin/bash -c "PLUGIN_PATH=$PLUGIN_PATH PLUGIN_AVAILABLE_PATH=$PLUGIN_AVAILABLE_PATH PLUGIN_ENABLED_PATH=$PLUGIN_ENABLED_PATH PLUGIN_CORE_AVAILABLE_PATH=$PLUGIN_CORE_AVAILABLE_PATH DOKKU_LIB_ROOT=$DOKKU_LIB_ROOT $PLUGIN_ENABLED_PATH/$PLUGIN_NAME/$SCRIPT_NAME $*"
|
||||
}
|
||||
|
||||
# Hold the deploy lock for $TEST_APP from a background process, as an in-progress deploy would.
|
||||
# Call release_deploy_lock (typically from teardown) to release it.
|
||||
hold_deploy_lock() {
|
||||
DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
|
||||
DEPLOY_LOCK_HOLDER_PID_FILE="$(mktemp "/tmp/${DOKKU_DOMAIN}.XXXXX")"
|
||||
chown dokku:dokku "$DEPLOY_LOCK_HOLDER_PID_FILE"
|
||||
|
||||
# the holder writes a build id like a deploy would, then becomes the sleep
|
||||
# process so that killing its pid releases the lock
|
||||
sudo -u dokku bash -c "exec 200>>'$DEPLOY_LOCK_FILE'; flock -n 200 || exit 1; echo test-lock-holder >'$DEPLOY_LOCK_FILE'; echo \$\$ >'$DEPLOY_LOCK_HOLDER_PID_FILE'; exec sleep 120" >/dev/null 2>&1 3>&- &
|
||||
|
||||
local i
|
||||
for i in $(seq 1 20); do
|
||||
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]] && ! flock -n "$DEPLOY_LOCK_FILE" true; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "unable to acquire the deploy lock for $TEST_APP"
|
||||
return 1
|
||||
}
|
||||
|
||||
release_deploy_lock() {
|
||||
if [[ -z "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local holder_pid i
|
||||
if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
|
||||
holder_pid="$(cat "$DEPLOY_LOCK_HOLDER_PID_FILE")"
|
||||
kill "$holder_pid" &>/dev/null || true
|
||||
for i in $(seq 1 20); do
|
||||
kill -0 "$holder_pid" &>/dev/null || break
|
||||
sleep 0.5
|
||||
done
|
||||
fi
|
||||
rm -f "$DEPLOY_LOCK_FILE" "$DEPLOY_LOCK_HOLDER_PID_FILE"
|
||||
DEPLOY_LOCK_HOLDER_PID_FILE=""
|
||||
}
|
||||
|
||||
check_urls() {
|
||||
local PATTERN="$1"
|
||||
run /bin/bash -c "dokku urls $TEST_APP | grep -E \"${PATTERN}\""
|
||||
|
||||
Reference in new issue
Block a user