Merge pull request #9131 from dokku/fix-source-image-on-failed-deploy

Keep the git source-image property in sync and harden the deploy lock

docs/deployment/methods/git.md

@@ -170,6 +170,8 @@ By default, when running `git:sync` with a git branch reference, Dokku will auto
 dokku git:sync --skip-deploy-branch node-js-app https://github.com/heroku/node-js-getting-started.git main
 ```
 
+The `git:sync` command holds the app's deploy lock while fetching, and will fail if a deploy is already in progress. When the fetched code changes the repository or a build is triggered, the `source-image` property set by [image deploys](/docs/deployment/methods/image.md) is cleared. As the synced code is kept even when a build fails, the property is not restored on build failure.
+
 ### Initializing from private repositories
 
 > [!IMPORTANT]

docs/deployment/methods/image.md

@@ -59,6 +59,8 @@ dokku git:from-image --build-dir path/to/build node-js-app domy-registrykku/node
 
 See the [dockerfile documentation](/docs/deployment/builders/dockerfiles.md) to learn about the different ways to configure Dockerfile-based deploys.
 
+The image used for a `git:from-image` or `git:load-image` deploy is stored in the `source-image` git property, which can be viewed via `dokku git:report node-js-app --git-source-image`. This property is only changed while the app's deploy lock is held. If the deploy fails, the deploy branch is rolled back and the property is restored to its previous value, so it always refers to the image the repository was last successfully deployed from. Deploying via other methods - such as `git push`, `git:from-archive`, or `git:sync` - clears the property.
+
 ## Initializing an app repository from a remote image without a registry
 
 > [!IMPORTANT]

plugins/apps/subcommands.go

@@ -137,8 +137,13 @@ func CommandLock(appName string) error {
 		return err
 	}
 
+	// the file is not truncated as it may hold the build id of an in-progress deploy
 	lockPath := getLockPath(appName)
-	if _, err := os.Create(lockPath); err != nil {
+	lockFile, err := os.OpenFile(lockPath, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return errors.New("Unable to create deploy lock")
+	}
+	if err := lockFile.Close(); err != nil {
 		return errors.New("Unable to create deploy lock")
 	}
 

plugins/common/functions

@@ -980,11 +980,17 @@ dokku_discard_build_capture() {
   unset DOKKU_BUILD_ID DOKKU_BUILD_LOG_FILE DOKKU_BUILD_SOURCE
 }
 
+app_deploy_lock_file() {
+  declare desc="outputs the path to the deploy lock file for an app"
+  declare APP="$1"
+  echo "$DOKKU_LIB_ROOT/data/apps/$APP/.deploy.lock"
+}
+
 acquire_app_deploy_lock() {
   declare desc="acquire advisory lock for use in deploys"
   local APP="$1"
   local LOCK_TYPE="${2:-waiting}"
-  local APP_DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$APP/.deploy.lock"
+  local APP_DEPLOY_LOCK_FILE="$(app_deploy_lock_file "$APP")"
   local LOCK_WAITING_MSG="$APP currently has a deploy lock in place. Waiting..."
   local LOCK_FAILED_MSG="$APP currently has a deploy lock in place. Exiting..."
 
@@ -1003,7 +1009,7 @@ release_app_deploy_lock() {
   declare desc="release advisory lock used in deploys"
   local APP="$1"
   local EXIT_CODE="${2:-0}"
-  local APP_DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$APP/.deploy.lock"
+  local APP_DEPLOY_LOCK_FILE="$(app_deploy_lock_file "$APP")"
 
   if [[ -n "$DOKKU_BUILD_ID" ]]; then
     plugn trigger builds-record-finalize "$APP" "$DOKKU_BUILD_ID" "$EXIT_CODE" || true
@@ -1018,24 +1024,35 @@ acquire_advisory_lock() {
   local LOCK_FD="200"
   local SHOW_MSG=true
 
-  eval "exec $LOCK_FD>$LOCK_FILE"
+  while true; do
-  if [[ "$LOCK_TYPE" == "waiting" ]]; then
+    # opened for appending so a failed attempt does not truncate the build id
-    while [[ $(
+    # written by the current lock holder
-      flock -n "$LOCK_FD" &>/dev/null
+    eval "exec $LOCK_FD>>$LOCK_FILE"
-      echo $?
+    if [[ "$LOCK_TYPE" == "waiting" ]]; then
-    ) -ne 0 ]]; do
+      while [[ $(
-      if [[ "$SHOW_MSG" == "true" ]]; then
+        flock -n "$LOCK_FD" &>/dev/null
-        echo "$LOCK_WAITING_MSG"
+        echo $?
-        SHOW_MSG=false
+      ) -ne 0 ]]; do
+        if [[ "$SHOW_MSG" == "true" ]]; then
+          echo "$LOCK_WAITING_MSG"
+          SHOW_MSG=false
+        fi
+        sleep 1
+      done
+    else
+      if ! flock -n "$LOCK_FD" &>/dev/null; then
+        dokku_log_warn "$LOCK_FAILED_MSG"
+        dokku_log_fail "Run 'apps:unlock' to release the existing deploy lock"
       fi
-      sleep 1
-    done
-  else
-    if ! flock -n "$LOCK_FD" &>/dev/null; then
-      dokku_log_warn "$LOCK_FAILED_MSG"
-      dokku_log_fail "Run 'apps:unlock' to release the existing deploy lock"
     fi
-  fi
+
+    # the previous holder removes the lock file on release, so the file locked
+    # above may no longer be the one at the lock path
+    if [[ "/proc/self/fd/$LOCK_FD" -ef "$LOCK_FILE" ]]; then
+      break
+    fi
+    eval "exec $LOCK_FD>&-"
+  done
   echo "${DOKKU_BUILD_ID:-$DOKKU_PID}" >"$LOCK_FILE"
 }
 
@@ -1044,7 +1061,13 @@ release_advisory_lock() {
   local LOCK_FILE="$1"
   local LOCK_FD="200"
 
-  flock -u "$LOCK_FD" && rm -f "$LOCK_FILE" &>/dev/null
+  # the file is removed before unlocking so a waiter that locks it afterwards
+  # can tell that its lock is stale. It is only removed if it is still the file
+  # this process locked, as apps:unlock may have handed the path to another deploy
+  if [[ "/proc/self/fd/$LOCK_FD" -ef "$LOCK_FILE" ]]; then
+    rm -f "$LOCK_FILE" &>/dev/null
+  fi
+  flock -u "$LOCK_FD"
 }
 
 suppress_output() {

plugins/git/functions

@@ -88,12 +88,12 @@ cmd-git-hook() {
     # Only run this script for the master branch. You can remove this
     # if block if you wish to run it for others as well.
     if [[ $refname == "refs/heads/${DOKKU_DEPLOY_BRANCH}" ]] || [[ $refname == "refs/tags/${DOKKU_DEPLOY_BRANCH}" ]]; then
-      git_receive_app "$APP" "$newrev"
+      DOKKU_GIT_SOURCE_IMAGE="" git_receive_app "$APP" "$newrev"
       plugn trigger deploy-source-set "$APP" "git-push" "$newrev"
     else
       # broken out into plugin so we might support other methods to receive an app
       if [[ $(find "$PLUGIN_PATH"/enabled/*/receive-branch 2>/dev/null | wc -l) != 1 ]]; then
-        plugn trigger receive-branch "$APP" "$newrev" "$refname"
+        DOKKU_GIT_SOURCE_IMAGE="" plugn trigger receive-branch "$APP" "$newrev" "$refname"
         plugn trigger deploy-source-set "$APP" "git-push" "$newrev"
       elif [[ -z "$(fn-git-computed-deploy-branch "$APP" "")" ]]; then
         local deploy_branch="${refname/refs\/heads\//}"
@@ -101,7 +101,7 @@ cmd-git-hook() {
 
         echo $'\e[1G\e[K'"-----> Set ${deploy_branch} as deploy-branch"
         fn-plugin-property-write "git" "$APP" "deploy-branch" "${deploy_branch}"
-        git_receive_app "$APP" "$newrev"
+        DOKKU_GIT_SOURCE_IMAGE="" git_receive_app "$APP" "$newrev"
         plugn trigger deploy-source-set "$APP" "git-push" "$newrev"
       else
         echo $'\e[1G\e[K'"-----> WARNING: deploy did not complete, you must push to ${DOKKU_DEPLOY_BRANCH}."
@@ -152,12 +152,34 @@ git_receive_app() {
   else
     local exit_code=0
     acquire_app_deploy_lock "$APP" "exclusive"
-    git_build "$APP" "$REV" || exit_code="$?"
+    fn-git-build-with-lock-held "$APP" "$REV" || exit_code="$?"
     release_app_deploy_lock "$APP" "$exit_code"
     return "$exit_code"
   fi
 }
 
+fn-git-build-with-lock-held() {
+  declare desc="builds an app at a revision, expecting the caller to hold the app deploy lock"
+  declare APP="$1" REV="$2"
+  local exit_code=0 has_source_image=false source_image previous_source_image
+
+  # DOKKU_GIT_SOURCE_IMAGE is only applied while the deploy lock is held. An
+  # empty value clears the property, while an unset variable leaves it as-is.
+  if [[ -n "${DOKKU_GIT_SOURCE_IMAGE+x}" ]]; then
+    has_source_image=true
+    source_image="$DOKKU_GIT_SOURCE_IMAGE"
+    unset DOKKU_GIT_SOURCE_IMAGE
+    previous_source_image="$(fn-git-source-image "$APP")"
+    fn-plugin-property-write "git" "$APP" "source-image" "$source_image"
+  fi
+
+  (git_build "$APP" "$REV") || exit_code="$?"
+  if [[ "$exit_code" -ne 0 ]] && [[ "$has_source_image" == "true" ]]; then
+    fn-plugin-property-write "git" "$APP" "source-image" "$previous_source_image"
+  fi
+  return "$exit_code"
+}
+
 cmd-git-upload-pack() {
   declare desc="executes git-upload-pack"
   declare cmd="git-upload-pack"

plugins/git/git-from-archive

@@ -51,7 +51,7 @@ trigger-git-git-from-archive() {
     popd &>/dev/null || pushd "/tmp" >/dev/null
   fi
 
-  plugn trigger git-from-directory "$APP" "$TMP_WORK_DIR" "$USER_NAME" "$USER_EMAIL"
+  DOKKU_GIT_SOURCE_IMAGE="" plugn trigger git-from-directory "$APP" "$TMP_WORK_DIR" "$USER_NAME" "$USER_EMAIL"
 }
 
 trigger-git-git-from-archive "$@"

plugins/git/git-from-directory

@@ -5,20 +5,19 @@ source "$PLUGIN_AVAILABLE_PATH/git/internal-functions"
 set -eo pipefail
 [[ $DOKKU_TRACE ]] && set -x
 
-trigger-git-git-from-directory() {
+fn-git-from-directory-with-lock-held() {
-  declare desc="updates a repository from a directory"
+  declare desc="updates a repository from a directory and builds it, expecting the caller to hold the app deploy lock"
-  declare trigger="git-from-directory"
+  declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="$3" USER_EMAIL="$4"
-  declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="${3:-Dokku}" USER_EMAIL="${4:-automated@dokku.sh}"
 
   local APP_ROOT="$DOKKU_ROOT/$APP"
   local DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
-  local REV
+  local REV PREVIOUS_REV
   local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
   local TMP_WORK_DIR_2=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
   trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT
 
   local has_code=false
-  if fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then
+  if PREVIOUS_REV="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null)"; then
     has_code=true
   fi
 
@@ -60,18 +59,39 @@ trigger-git-git-from-directory() {
   fi
 
   rm -rf "$TMP_WORK_DIR" "$TMP_WORK_DIR_2" >/dev/null
-  if git_receive_app "$APP" "$REV"; then
+  if fn-git-build-with-lock-held "$APP" "$REV"; then
     return
   else
     local exit_code=$?
     local DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
+    # only roll back the deploy branch if it still points at the commit created above
     if [[ "$has_code" == "false" ]]; then
-      fn-git-cmd "$APP_ROOT" update-ref -d HEAD
+      fn-git-cmd "$APP_ROOT" update-ref -d "refs/heads/$DOKKU_DEPLOY_BRANCH" "$REV" || dokku_log_warn "Unable to roll back the $DOKKU_DEPLOY_BRANCH branch"
     else
-      fn-git-cmd "$APP_ROOT" update-ref "refs/heads/$DOKKU_DEPLOY_BRANCH" "$DOKKU_DEPLOY_BRANCH^"
+      fn-git-cmd "$APP_ROOT" update-ref "refs/heads/$DOKKU_DEPLOY_BRANCH" "$PREVIOUS_REV" "$REV" || dokku_log_warn "Unable to roll back the $DOKKU_DEPLOY_BRANCH branch"
     fi
     return "$exit_code"
   fi
 }
 
+trigger-git-git-from-directory() {
+  declare desc="updates a repository from a directory"
+  declare trigger="git-from-directory"
+  declare APP="$1" SOURCECODE_WORK_DIR="$2" USER_NAME="${3:-Dokku}" USER_EMAIL="${4:-automated@dokku.sh}"
+  local exit_code=0
+
+  # the lock is held from before the deploy branch is updated until after it is
+  # rolled back, and the subshell ensures it is released on any failure
+  acquire_app_deploy_lock "$APP" "exclusive"
+  set +e
+  (
+    set -e
+    fn-git-from-directory-with-lock-held "$APP" "$SOURCECODE_WORK_DIR" "$USER_NAME" "$USER_EMAIL"
+  )
+  exit_code="$?"
+  set -e
+  release_app_deploy_lock "$APP" "$exit_code"
+  return "$exit_code"
+}
+
 trigger-git-git-from-directory "$@"

plugins/git/git-from-image

@@ -44,8 +44,7 @@ trigger-git-git-from-image() {
     dokku_log_info1 "Image exists on host, skipping pull"
   fi
 
-  fn-plugin-property-write "git" "$APP" "source-image" "$DOCKER_IMAGE"
+  DOKKU_GIT_SOURCE_IMAGE="$DOCKER_IMAGE" plugn trigger git-from-directory "$APP" "$TMP_WORK_DIR" "$USER_NAME" "$USER_EMAIL"
-  plugn trigger git-from-directory "$APP" "$TMP_WORK_DIR" "$USER_NAME" "$USER_EMAIL"
 }
 
 trigger-git-git-from-image "$@"

plugins/git/internal-functions

@@ -279,33 +279,59 @@ cmd-git-sync() {
   fi
 
   local APP_ROOT="$DOKKU_ROOT/$APP"
-  DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
+  local APP_DEPLOY_LOCK_FILE="$(app_deploy_lock_file "$APP")"
-  CURRENT_REF="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null || true)"
+  local fetch_exit_code=0
 
   dokku_setup_build_capture "$APP" "git:sync"
 
-  if ! fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then
+  # acquire_app_deploy_lock is not used as releasing it would finalize the
-    dokku_log_info1_quiet "Cloning $APP from $GIT_REMOTE#$GIT_REF"
+  # build record before the build starts
-    fn-git-clone "$APP" "$GIT_REMOTE" "$GIT_REF" "$SKIP_DEPLOY_BRANCH"
+  acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "exclusive" "$APP currently has a deploy lock in place. Waiting..." "$APP currently has a deploy lock in place. Exiting..."
-  else
+
-    dokku_log_verbose "Fetching remote code for $APP from $GIT_REMOTE#$GIT_REF"
+  DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
-    fn-git-fetch "$APP" "$GIT_REMOTE" "$GIT_REF"
+  CURRENT_REF="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null || true)"
+
+  # the subshell keeps set -e active and ensures the lock is released on failure
+  set +e
+  (
+    set -e
+    if ! fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then
+      dokku_log_info1_quiet "Cloning $APP from $GIT_REMOTE#$GIT_REF"
+      fn-git-clone "$APP" "$GIT_REMOTE" "$GIT_REF" "$SKIP_DEPLOY_BRANCH"
+    else
+      dokku_log_verbose "Fetching remote code for $APP from $GIT_REMOTE#$GIT_REF"
+      fn-git-fetch "$APP" "$GIT_REMOTE" "$GIT_REF"
+    fi
+  )
+  fetch_exit_code="$?"
+  set -e
+
+  if [[ "$fetch_exit_code" -ne 0 ]]; then
+    release_advisory_lock "$APP_DEPLOY_LOCK_FILE"
+    return "$fetch_exit_code"
   fi
 
   UPDATED_REF="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null || true)"
   local SHOULD_BUILD=false
   if [[ "$FLAG" == "--build" ]]; then
     SHOULD_BUILD=true
-  elif [[ "$FLAG" == "--build-if-changes" ]]; then
+  elif [[ "$FLAG" == "--build-if-changes" ]] && [[ "$CURRENT_REF" != "$UPDATED_REF" ]]; then
-    if [[ "$CURRENT_REF" == "$UPDATED_REF" ]]; then
-      dokku_log_verbose "Skipping build as no changes were detected"
-      dokku_discard_build_capture "$APP"
-      return
-    fi
-
     SHOULD_BUILD=true
   fi
 
+  # the synced code is kept even if the build fails, so the source-image is
+  # cleared rather than restored on failure
+  if [[ "$CURRENT_REF" != "$UPDATED_REF" ]] || [[ "$SHOULD_BUILD" == "true" ]]; then
+    fn-plugin-property-write "git" "$APP" "source-image"
+  fi
+  release_advisory_lock "$APP_DEPLOY_LOCK_FILE"
+
+  if [[ "$FLAG" == "--build-if-changes" ]] && [[ "$SHOULD_BUILD" == "false" ]]; then
+    dokku_log_verbose "Skipping build as no changes were detected"
+    dokku_discard_build_capture "$APP"
+    return
+  fi
+
   if [[ "$SHOULD_BUILD" == "true" ]]; then
     if [[ -n "$GIT_REF" ]]; then
       GIT_REF="$(fn-git-cmd "$APP_ROOT" rev-parse "$GIT_REF")"

tests/unit/apps_1.bats

@@ -402,6 +402,36 @@ teardown() {
   destroy_app
 }
 
+@test "(apps) apps:lock does not truncate an existing lock file" {
+  run create_app
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "echo holder-id >$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock && chown dokku:dokku $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku apps:lock $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat $DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "holder-id"
+
+  run /bin/bash -c "dokku apps:unlock $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  destroy_app
+}
+
 @test "(apps:report) emits new stripped JSON keys alongside legacy" {
   create_app
 

tests/unit/deploy-lock.bats

@@ -0,0 +1,109 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  global_setup
+  LOCK_DIR="$(mktemp -d "/tmp/${DOKKU_DOMAIN}.XXXXX")"
+  LOCK_FILE="$LOCK_DIR/test.lock"
+  LOCK_PIDS=()
+}
+
+teardown() {
+  local pid
+  touch "$LOCK_DIR/release"
+  for pid in "${LOCK_PIDS[@]}"; do
+    kill "$pid" &>/dev/null || true
+  done
+  rm -rf "$LOCK_DIR"
+  global_teardown
+}
+
+lock_script() {
+  declare BUILD_ID="$1" SCRIPT="$2"
+  echo "source '$PLUGIN_CORE_AVAILABLE_PATH/common/functions'; export DOKKU_BUILD_ID='$BUILD_ID'; $SCRIPT"
+}
+
+start_lock_holder() {
+  declare BUILD_ID="$1"
+  /bin/bash -c "$(lock_script "$BUILD_ID" "acquire_advisory_lock '$LOCK_FILE' exclusive 'waiting' 'lock held'; while [[ ! -f '$LOCK_DIR/release' ]]; do sleep 0.2; done; release_advisory_lock '$LOCK_FILE'")" >/dev/null 2>&1 3>&- &
+  LOCK_PIDS+=("$!")
+  wait_for "[[ \"\$(cat '$LOCK_FILE' 2>/dev/null)\" == '$BUILD_ID' ]]"
+}
+
+wait_for() {
+  declare CONDITION="$1"
+  local i
+  for i in $(seq 1 50); do
+    if eval "$CONDITION"; then
+      return 0
+    fi
+    sleep 0.2
+  done
+
+  echo "timed out waiting for: $CONDITION"
+  return 1
+}
+
+@test "(deploy-lock) a failed acquire does not truncate the lock holder's build id" {
+  start_lock_holder "holder-a"
+
+  run /bin/bash -c "$(lock_script "other" "acquire_advisory_lock '$LOCK_FILE' exclusive 'waiting' 'lock held'")"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+  assert_output_contains "lock held"
+
+  run /bin/bash -c "cat '$LOCK_FILE'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "holder-a"
+}
+
+@test "(deploy-lock) a waiter does not keep a stale lock after the holder removes the lock file" {
+  start_lock_holder "holder-a"
+
+  /bin/bash -c "$(lock_script "holder-b" "acquire_advisory_lock '$LOCK_FILE' waiting 'waiting' 'lock held'; touch '$LOCK_DIR/b.acquired'; exec sleep 60")" >"$LOCK_DIR/b.log" 2>&1 3>&- &
+  local B_PID="$!"
+  LOCK_PIDS+=("$B_PID")
+  wait_for "grep -q waiting '$LOCK_DIR/b.log'"
+
+  touch "$LOCK_DIR/release"
+  wait_for "[[ -f '$LOCK_DIR/b.acquired' ]]"
+
+  run /bin/bash -c "[[ /proc/$B_PID/fd/200 -ef '$LOCK_FILE' ]]"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat '$LOCK_FILE'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "holder-b"
+
+  run /bin/bash -c "$(lock_script "holder-c" "acquire_advisory_lock '$LOCK_FILE' exclusive 'waiting' 'lock held'")"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+  assert_output_contains "lock held"
+}
+
+@test "(deploy-lock) release only removes the lock file it holds" {
+  start_lock_holder "holder-a"
+
+  run /bin/bash -c "rm -f '$LOCK_FILE' && echo 'replacement' >'$LOCK_FILE'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  touch "$LOCK_DIR/release"
+  wait_for "! kill -0 ${LOCK_PIDS[0]} 2>/dev/null"
+
+  run /bin/bash -c "cat '$LOCK_FILE'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "replacement"
+}

tests/unit/git_4.bats

@@ -10,11 +10,88 @@ setup() {
 }
 
 teardown() {
+  release_deploy_lock
+  docker image rm "dokku-test/$TEST_APP:second-procfile" "dokku-test/$TEST_APP:default-procfile" &>/dev/null || true
   rm -f /home/dokku/.ssh/id_rsa.pub || true
   destroy_app
   global_teardown
 }
 
+build_python_image() {
+  declare TAG="$1" PROCFILE="$2"
+  local TMP_IMAGE_DIR exit_code=0
+  TMP_IMAGE_DIR="$(mktemp -d "/tmp/${DOKKU_DOMAIN}.XXXXX")"
+  cp -r "${BATS_TEST_DIRNAME}/../../tests/apps/python/." "$TMP_IMAGE_DIR"
+  if [[ -n "$PROCFILE" ]]; then
+    cp "$TMP_IMAGE_DIR/$PROCFILE" "$TMP_IMAGE_DIR/Procfile"
+  fi
+
+  docker image build -t "dokku-test/$TEST_APP:$TAG" -f "$TMP_IMAGE_DIR/alt.Dockerfile" "$TMP_IMAGE_DIR" || exit_code="$?"
+  rm -rf "$TMP_IMAGE_DIR"
+  return "$exit_code"
+}
+
+deploy_second_procfile_image() {
+  run build_python_image second-procfile second.Procfile
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP dokku-test/$TEST_APP:second-procfile"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku plugin:trigger procfile-get-command $TEST_APP web 5001"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "second.Procfile"
+}
+
+hold_deploy_lock() {
+  DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/apps/$TEST_APP/.deploy.lock"
+  DEPLOY_LOCK_HOLDER_PID_FILE="$(mktemp "/tmp/${DOKKU_DOMAIN}.XXXXX")"
+  chown dokku:dokku "$DEPLOY_LOCK_HOLDER_PID_FILE"
+
+  # the holder writes a build id like a deploy would, then becomes the sleep
+  # process so that killing its pid releases the lock
+  sudo -u dokku bash -c "exec 200>>'$DEPLOY_LOCK_FILE'; flock -n 200 || exit 1; echo test-lock-holder >'$DEPLOY_LOCK_FILE'; echo \$\$ >'$DEPLOY_LOCK_HOLDER_PID_FILE'; exec sleep 120" >/dev/null 2>&1 3>&- &
+
+  local i
+  for i in $(seq 1 20); do
+    if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]] && ! flock -n "$DEPLOY_LOCK_FILE" true; then
+      return 0
+    fi
+    sleep 0.5
+  done
+
+  echo "unable to acquire the deploy lock for $TEST_APP"
+  return 1
+}
+
+release_deploy_lock() {
+  if [[ -z "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
+    return 0
+  fi
+
+  local holder_pid i
+  if [[ -s "$DEPLOY_LOCK_HOLDER_PID_FILE" ]]; then
+    holder_pid="$(cat "$DEPLOY_LOCK_HOLDER_PID_FILE")"
+    kill "$holder_pid" &>/dev/null || true
+    for i in $(seq 1 20); do
+      kill -0 "$holder_pid" &>/dev/null || break
+      sleep 0.5
+    done
+  fi
+  rm -f "$DEPLOY_LOCK_FILE" "$DEPLOY_LOCK_HOLDER_PID_FILE"
+  DEPLOY_LOCK_HOLDER_PID_FILE=""
+}
+
+deploy_branch_rev() {
+  sudo -u dokku git -C "/home/dokku/$TEST_APP" rev-parse master
+}
+
 @test "(git) git:from-image [missing]" {
   run /bin/bash -c "dokku git:from-image $TEST_APP dokku/python:missing-tag"
   echo "output: $output"
@@ -50,11 +127,21 @@ teardown() {
   assert_output_contains "Image exists on host, skipping pull"
   assert_output_contains "No changes detected, skipping git commit"
 
+  run /bin/bash -c "dokku apps:locked $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
   run /bin/bash -c "dokku git:from-image --force $TEST_APP linuxserver/foldingathome:7.6.21"
   echo "output: $output"
   echo "status: $status"
   assert_output_contains "Force pulling image"
   assert_output_contains "No changes detected, skipping git commit"
+
+  run /bin/bash -c "dokku apps:locked $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
 }
 
 @test "(git) git:from-image [normal-custom-branch]" {
@@ -141,6 +228,17 @@ teardown() {
   assert_failure
   assert_output "fatal: this operation must be run in a work tree"
 
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "dokku apps:locked $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
   run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
   echo "output: $output"
   echo "status: $status"
@@ -151,6 +249,12 @@ teardown() {
   echo "status: $status"
   assert_success
 
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:latest"
+
   run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=true"
   echo "output: $output"
   echo "status: $status"
@@ -161,6 +265,17 @@ teardown() {
   echo "status: $status"
   assert_failure
 
+  run /bin/bash -c "dokku apps:locked $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:latest"
+
   run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
   echo "output: $output"
   echo "status: $status"
@@ -172,6 +287,12 @@ teardown() {
   assert_success
   assert_output_contains "No changes detected, skipping git commit" 0
 
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:v2"
+
   run /bin/bash -c "docker image rm dokku-test/$TEST_APP:latest dokku-test/$TEST_APP:v2"
   echo "output: $output"
   echo "status: $status"
@@ -249,3 +370,235 @@ EOF
   assert_success
   assert_output_contains "linuxserver/foldingathome:7.6.21"
 }
+
+@test "(git) git:from-image source-image after failed deploy and ps:rebuild" {
+  deploy_second_procfile_image
+
+  run build_python_image default-procfile
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=true"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP dokku-test/$TEST_APP:default-procfile"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:second-procfile"
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:rebuild $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:second-procfile"
+
+  run /bin/bash -c "dokku plugin:trigger procfile-get-command $TEST_APP web 5001"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "second.Procfile"
+  assert_output_contains "first.Procfile" 0
+}
+
+@test "(git) git:from-image then git:from-archive ignores the previous source-image" {
+  deploy_second_procfile_image
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=true"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "tar -C ${BATS_TEST_DIRNAME}/../../tests/apps -cf - python | dokku git:from-archive $TEST_APP --"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:second-procfile"
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "tar -C ${BATS_TEST_DIRNAME}/../../tests/apps -cf - python | dokku git:from-archive $TEST_APP --"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "dokku plugin:trigger procfile-get-command $TEST_APP web 5001"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "first.Procfile"
+  assert_output_contains "second.Procfile" 0
+}
+
+@test "(git) git:from-image then git push ignores the previous source-image" {
+  local TMP=$(mktemp -d "/tmp/${DOKKU_DOMAIN}.XXXXX")
+  trap 'rm -rf "$TMP"' INT TERM
+
+  deploy_second_procfile_image
+
+  rmdir "$TMP" && cp -r "${BATS_TEST_DIRNAME}/../../tests/apps/python" "$TMP"
+  run /bin/bash -c "cd $TMP && git init -q && git checkout -q -b master && git config user.email 'robot@example.com' && git config user.name 'Test Robot' && git add . && git commit -q -m 'initial commit' && git push --force dokku@${DOKKU_DOMAIN}:$TEST_APP master:master"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  rm -rf "$TMP"
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "dokku plugin:trigger procfile-get-command $TEST_APP web 5001"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "first.Procfile"
+  assert_output_contains "second.Procfile" 0
+}
+
+@test "(git) git:from-image then git:sync clears the source-image" {
+  deploy_second_procfile_image
+
+  run /bin/bash -c "dokku git:sync $TEST_APP https://github.com/dokku/smoke-test-app.git 2.0.0"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP dokku-test/$TEST_APP:second-procfile"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:second-procfile"
+
+  run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git 2.0.0"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+}
+
+@test "(git) source-image is unchanged when the deploy lock is held" {
+  deploy_second_procfile_image
+
+  run build_python_image default-procfile
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  local DEPLOY_REV
+  DEPLOY_REV="$(deploy_branch_rev)"
+  echo "deploy branch rev: $DEPLOY_REV"
+
+  hold_deploy_lock
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP dokku-test/$TEST_APP:default-procfile"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+  assert_output_contains "currently has a deploy lock in place"
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:second-procfile"
+
+  run deploy_branch_rev
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$DEPLOY_REV"
+
+  run /bin/bash -c "cat $DEPLOY_LOCK_FILE"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "test-lock-holder"
+
+  run /bin/bash -c "dokku git:sync $TEST_APP https://github.com/dokku/smoke-test-app.git 2.0.0"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+  assert_output_contains "currently has a deploy lock in place"
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:second-procfile"
+
+  run deploy_branch_rev
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$DEPLOY_REV"
+
+  run /bin/bash -c "cat $DEPLOY_LOCK_FILE"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "test-lock-holder"
+
+  release_deploy_lock
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP dokku-test/$TEST_APP:default-procfile"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "No changes detected, skipping git commit" 0
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:default-procfile"
+}

tests/unit/git_6.bats

@@ -154,6 +154,12 @@ teardown() {
   assert_failure
   assert_output "fatal: this operation must be run in a work tree"
 
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
   run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
   echo "output: $output"
   echo "status: $status"
@@ -164,6 +170,12 @@ teardown() {
   echo "status: $status"
   assert_success
 
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:latest"
+
   run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=true"
   echo "output: $output"
   echo "status: $status"
@@ -174,6 +186,12 @@ teardown() {
   echo "status: $status"
   assert_failure
 
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:latest"
+
   run /bin/bash -c "dokku config:set --no-restart $TEST_APP FAIL_ON_STARTUP=false"
   echo "output: $output"
   echo "status: $status"
@@ -184,6 +202,12 @@ teardown() {
   echo "status: $status"
   assert_success
   assert_output_contains "No changes detected, skipping git commit" 0
+
+  run /bin/bash -c "dokku git:report $TEST_APP --git-source-image"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dokku-test/$TEST_APP:v2"
 }
 
 @test "(git) git:load-image [onbuild]" {