fix: hold the deploy lock across git-from-directory and harden the advisory lock

git-from-directory updated the deploy branch before taking the deploy
lock and rolled it back after releasing it. The lock is now held for the
whole update, build, and rollback, and the rollback only moves the branch
if it still points at the commit that was created.

A failed lock attempt and apps:lock both truncated the lock file,
dropping the build id of the in-progress deploy that builds:cancel and
builds:output read. Releasing the lock unlocked before removing the
file, so a waiter could end up holding a lock on a deleted file while
another deploy locked a new one. The file is now removed before
unlocking, and acquiring retries when the locked file is no longer the
one at the lock path.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-10-11 16:21:13 -04:00
1 parent 7955e8ee06
commit 11c4e908d5
8 files changed
+312 -56

No files matched your search

+24 -16
View File
@@ -150,28 +150,36 @@ git_receive_app() {
if [[ ! -d "$DOKKU_ROOT/$APP/refs" ]]; then
true
else
local exit_code=0 has_source_image=false source_image previous_source_image
local exit_code=0
acquire_app_deploy_lock "$APP" "exclusive"
# DOKKU_GIT_SOURCE_IMAGE is only applied while the deploy lock is held. An
# empty value clears the property, while an unset variable leaves it as-is.
if [[ -n "${DOKKU_GIT_SOURCE_IMAGE+x}" ]]; then
has_source_image=true
source_image="$DOKKU_GIT_SOURCE_IMAGE"
unset DOKKU_GIT_SOURCE_IMAGE
previous_source_image="$(fn-git-source-image "$APP")"
fn-plugin-property-write "git" "$APP" "source-image" "$source_image"
fi
(git_build "$APP" "$REV") || exit_code="$?"
if [[ "$exit_code" -ne 0 ]] && [[ "$has_source_image" == "true" ]]; then
fn-plugin-property-write "git" "$APP" "source-image" "$previous_source_image"
fi
fn-git-build-with-lock-held "$APP" "$REV" || exit_code="$?"
release_app_deploy_lock "$APP" "$exit_code"
return "$exit_code"
fi
}
fn-git-build-with-lock-held() {
declare desc="builds an app at a revision, expecting the caller to hold the app deploy lock"
declare APP="$1" REV="$2"
local exit_code=0 has_source_image=false source_image previous_source_image
# DOKKU_GIT_SOURCE_IMAGE is only applied while the deploy lock is held. An
# empty value clears the property, while an unset variable leaves it as-is.
if [[ -n "${DOKKU_GIT_SOURCE_IMAGE+x}" ]]; then
has_source_image=true
source_image="$DOKKU_GIT_SOURCE_IMAGE"
unset DOKKU_GIT_SOURCE_IMAGE
previous_source_image="$(fn-git-source-image "$APP")"
fn-plugin-property-write "git" "$APP" "source-image" "$source_image"
fi
(git_build "$APP" "$REV") || exit_code="$?"
if [[ "$exit_code" -ne 0 ]] && [[ "$has_source_image" == "true" ]]; then
fn-plugin-property-write "git" "$APP" "source-image" "$previous_source_image"
fi
return "$exit_code"
}
cmd-git-upload-pack() {
declare desc="executes git-upload-pack"
declare cmd="git-upload-pack"
+29 -10
View File
@@ -5,20 +5,19 @@ source "$PLUGIN_AVAILABLE_PATH/git/internal-functions"
set -eo pipefail
[[ $DOKKU_TRACE ]] && set -x
trigger-git-git-from-directory() {
declare desc="updates a repository from a directory"
declare trigger="git-from-directory"
declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="${3:-Dokku}" USER_EMAIL="${4:-automated@dokku.sh}"
fn-git-from-directory-with-lock-held() {
declare desc="updates a repository from a directory and builds it, expecting the caller to hold the app deploy lock"
declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="$3" USER_EMAIL="$4"
local APP_ROOT="$DOKKU_ROOT/$APP"
local DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
local REV
local REV PREVIOUS_REV
local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
local TMP_WORK_DIR_2=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT
local has_code=false
if fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then
if PREVIOUS_REV="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null)"; then
has_code=true
fi
@@ -60,19 +59,39 @@ trigger-git-git-from-directory() {
fi
rm -rf "$TMP_WORK_DIR" "$TMP_WORK_DIR_2" >/dev/null
# run in a subshell so a failure to acquire the deploy lock still reaches the rollback
if (git_receive_app "$APP" "$REV"); then
if fn-git-build-with-lock-held "$APP" "$REV"; then
return
else
local exit_code=$?
local DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
# only roll back the deploy branch if it still points at the commit created above
if [[ "$has_code" == "false" ]]; then
fn-git-cmd "$APP_ROOT" update-ref -d HEAD
fn-git-cmd "$APP_ROOT" update-ref -d "refs/heads/$DOKKU_DEPLOY_BRANCH" "$REV" || dokku_log_warn "Unable to roll back the $DOKKU_DEPLOY_BRANCH branch"
else
fn-git-cmd "$APP_ROOT" update-ref "refs/heads/$DOKKU_DEPLOY_BRANCH" "$DOKKU_DEPLOY_BRANCH^"
fn-git-cmd "$APP_ROOT" update-ref "refs/heads/$DOKKU_DEPLOY_BRANCH" "$PREVIOUS_REV" "$REV" || dokku_log_warn "Unable to roll back the $DOKKU_DEPLOY_BRANCH branch"
fi
return "$exit_code"
fi
}
trigger-git-git-from-directory() {
declare desc="updates a repository from a directory"
declare trigger="git-from-directory"
declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="${3:-Dokku}" USER_EMAIL="${4:-automated@dokku.sh}"
local exit_code=0
# the lock is held from before the deploy branch is updated until after it is
# rolled back, and the subshell ensures it is released on any failure
acquire_app_deploy_lock "$APP" "exclusive"
set +e
(
set -e
fn-git-from-directory-with-lock-held "$APP" "$SOURCECODE_WORK_DIR" "$USER_NAME" "$USER_EMAIL"
)
exit_code="$?"
set -e
release_app_deploy_lock "$APP" "$exit_code"
return "$exit_code"
}
trigger-git-git-from-directory "$@"
+1 -1
View File
@@ -279,7 +279,7 @@ cmd-git-sync() {
fi
local APP_ROOT="$DOKKU_ROOT/$APP"
local APP_DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$APP/.deploy.lock"
local APP_DEPLOY_LOCK_FILE="$(app_deploy_lock_file "$APP")"
local fetch_exit_code=0
dokku_setup_build_capture "$APP" "git:sync"