fix: release deploy locks on exit and honor apps:lock

A deploy that exited before releasing its lock, such as after a failed build, left `.deploy.lock` behind, so `apps:locked` reported a lock nothing held and the build record stayed running. The lock is now released and the record finalized whenever the deploy exits. `apps:lock` never actually blocked deploys, as it only created the file deploys lock. It now creates its own lock that git pushes, `ps:rebuild`, `git:sync`, and the `git:from-*` commands refuse to deploy past, and `apps:locked` only reports a lock when the app was locked via `apps:lock` or a deploy is in progress. Locks created by `apps:lock` in older versions are migrated on install.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-10-24 13:58:47 -04:00
1 parent 57ae817dab
commit c4b164a270
17 files changed
+535 -69

No files matched your search

+35 -4
View File
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"os"
"syscall"
"time"
"github.com/dokku/dokku/plugins/common"
@@ -19,11 +20,9 @@ func appNameTaken(appName string) bool {
return common.DirectoryExists(common.AppRoot(appName))
}
// checks if an app is locked
// checks if an app is locked via apps:lock or by an in-progress deploy
func appIsLocked(appName string) bool {
lockPath := getLockPath(appName)
_, err := os.Stat(lockPath)
return !os.IsNotExist(err)
return appIsManuallyLocked(appName) || deployLockHeld(appName)
}
// verifies app name and creates an app
@@ -131,6 +130,38 @@ func getLockPath(appName string) string {
return fmt.Sprintf("%v/.deploy.lock", common.GetAppDataDirectory("apps", appName))
}
// returns the path of the lock created by apps:lock
func getManualLockPath(appName string) string {
return fmt.Sprintf("%v/.manual-deploy.lock", common.GetAppDataDirectory("apps", appName))
}
// checks if an app has been locked via apps:lock
func appIsManuallyLocked(appName string) bool {
_, err := os.Stat(getManualLockPath(appName))
return err == nil
}
// checks if a deploy currently holds the flock on the deploy lock file
//
// a shared lock is attempted and immediately released. Deploys taking the
// lock exclusively wait briefly before failing, so this check does not make
// a concurrent deploy fail.
func deployLockHeld(appName string) bool {
lockFile, err := os.Open(getLockPath(appName))
if err != nil {
return false
}
defer lockFile.Close()
fd := int(lockFile.Fd())
if err := syscall.Flock(fd, syscall.LOCK_SH|syscall.LOCK_NB); err != nil {
return errors.Is(err, syscall.EWOULDBLOCK)
}
_ = syscall.Flock(fd, syscall.LOCK_UN)
return false
}
// creates an app if allowed
func maybeCreateApp(appName string) error {
if err := appExists(appName); err == nil {
+7 -6
View File
@@ -137,9 +137,7 @@ func CommandLock(appName string) error {
return err
}
// the file is not truncated as it may hold the build id of an in-progress deploy
lockPath := getLockPath(appName)
lockFile, err := os.OpenFile(lockPath, os.O_CREATE|os.O_WRONLY, 0644)
lockFile, err := os.OpenFile(getManualLockPath(appName), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return errors.New("Unable to create deploy lock")
}
@@ -187,6 +185,10 @@ func CommandRename(oldAppName string, newAppName string, skipDeploy bool) error
return errors.New("Name is already taken")
}
if appIsManuallyLocked(oldAppName) {
return errors.New("Unable to rename a locked app, run apps:unlock first")
}
common.LogInfo1Quiet(fmt.Sprintf("Renaming %s to %s", oldAppName, newAppName))
if err := createApp(newAppName); err != nil {
return err
@@ -264,13 +266,12 @@ func CommandUnlock(appName string) error {
return err
}
lockfilePath := getLockPath(appName)
if _, err := os.Stat(lockfilePath); !os.IsNotExist(err) {
if deployLockHeld(appName) {
common.LogWarn("A deploy may be in progress.")
common.LogWarn("Removing the app lock will not stop in progress deploys.")
}
if err := os.Remove(lockfilePath); err != nil {
if err := removeLockFiles(getManualLockPath(appName), getLockPath(appName)); err != nil {
return errors.New("Unable to remove deploy lock")
}
+44 -2
View File
@@ -83,6 +83,40 @@ func TriggerInstall() error {
}
}
for _, appName := range apps {
if err := migrateManualLock(appName); err != nil {
return err
}
}
return nil
}
// migrateManualLock converts a lock created by apps:lock before manual locks
// had their own file. Deploys write their build id into the deploy lock file,
// so an empty one that no deploy holds was created by apps:lock.
func migrateManualLock(appName string) error {
lockPath := getLockPath(appName)
fi, err := os.Stat(lockPath)
if err != nil || fi.Size() != 0 || deployLockHeld(appName) {
return nil
}
if err := os.Rename(lockPath, getManualLockPath(appName)); err != nil {
return fmt.Errorf("Unable to migrate the deploy lock for %s: %w", appName, err)
}
return nil
}
// removeLockFiles removes the given lock files, ignoring ones that do not exist
func removeLockFiles(lockPaths ...string) error {
for _, lockPath := range lockPaths {
if err := os.Remove(lockPath); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
@@ -93,7 +127,11 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
return err
}
return common.CloneAppData("apps", oldAppName, newAppName)
if err := common.CloneAppData("apps", oldAppName, newAppName); err != nil {
return err
}
return removeLockFiles(getManualLockPath(newAppName), getLockPath(newAppName))
}
// TriggerPostAppRename removes the old app data
@@ -111,7 +149,11 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
return err
}
return common.CloneAppData("apps", oldAppName, newAppName)
if err := common.CloneAppData("apps", oldAppName, newAppName); err != nil {
return err
}
return removeLockFiles(getLockPath(newAppName))
}
// TriggerPostCreate ensures apps have the correct data directory structure
+24 -7
View File
@@ -319,15 +319,12 @@ func CommandCancel(appName string) error {
if err := WriteBuild(b); err != nil {
return err
}
_ = os.Remove(lockPath)
removeLockForBuild(lockPath, buildID)
return nil
}
common.LogInfo1("Killing app deploy")
if err := killProcessGroup(b.PID); err != nil {
common.LogWarn(fmt.Sprintf("Failed to signal process group %d: %s", b.PID, err))
}
// the record is finalized before the signal is sent so that a deploy
// finalizing itself on exit leaves it as canceled
current, err := ReadBuild(appName, buildID)
if err == nil && current.Status == BuildStatusRunning {
now := time.Now().UTC()
@@ -340,10 +337,30 @@ func CommandCancel(appName string) error {
}
}
_ = os.Remove(lockPath)
common.LogInfo1("Killing app deploy")
if err := killProcessGroup(b.PID); err != nil {
common.LogWarn(fmt.Sprintf("Failed to signal process group %d: %s", b.PID, err))
}
removeLockForBuild(lockPath, buildID)
return nil
}
// removeLockForBuild removes the deploy lock file if it still holds the given
// build id, leaving a lock taken by a newer deploy in place
func removeLockForBuild(lockPath string, buildID string) {
body, err := os.ReadFile(lockPath)
if err != nil {
return
}
if strings.TrimSpace(string(body)) != buildID {
return
}
_ = os.Remove(lockPath)
}
func killProcessGroup(pid int) error {
pgid, err := syscall.Getpgid(pid)
if err != nil {
+33 -6
View File
@@ -986,14 +986,35 @@ app_deploy_lock_file() {
echo "$DOKKU_LIB_ROOT/data/apps/$APP/.deploy.lock"
}
acquire_app_deploy_lock() {
declare desc="acquire advisory lock for use in deploys"
local APP="$1"
local LOCK_TYPE="${2:-waiting}"
app_manual_lock_file() {
declare desc="outputs the path to the manual lock file created by apps:lock for an app"
declare APP="$1"
echo "$DOKKU_LIB_ROOT/data/apps/$APP/.manual-deploy.lock"
}
acquire_app_advisory_lock() {
declare desc="acquire the advisory lock for an app's deploys, failing if the app was locked via apps:lock"
declare APP="$1" LOCK_TYPE="${2:-waiting}"
local APP_DEPLOY_LOCK_FILE="$(app_deploy_lock_file "$APP")"
local LOCK_WAITING_MSG="$APP currently has a deploy lock in place. Waiting..."
local LOCK_FAILED_MSG="$APP currently has a deploy lock in place. Exiting..."
acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG"
if [[ ! -f "$(app_manual_lock_file "$APP")" ]]; then
return 0
fi
release_advisory_lock "$APP_DEPLOY_LOCK_FILE"
dokku_discard_build_capture "$APP"
dokku_log_warn "$APP has been locked via apps:lock"
dokku_log_fail "Run 'apps:unlock' to allow deploys"
}
acquire_app_deploy_lock() {
declare desc="acquire advisory lock for use in deploys, releasing it and finalizing the build record when the shell exits"
local APP="$1"
local LOCK_TYPE="${2:-waiting}"
if [[ -z "$DOKKU_BUILD_ID" ]]; then
DOKKU_BUILD_ID="$(plugn trigger builds-generate-id 2>/dev/null)"
if [[ -z "$DOKKU_BUILD_ID" ]]; then
@@ -1002,7 +1023,8 @@ acquire_app_deploy_lock() {
export DOKKU_BUILD_ID
fi
acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG"
acquire_app_advisory_lock "$APP" "$LOCK_TYPE"
trap "release_app_deploy_lock '$APP' \"\$?\"" EXIT
}
release_app_deploy_lock() {
@@ -1040,7 +1062,9 @@ acquire_advisory_lock() {
sleep 1
done
else
if ! flock -n "$LOCK_FD" &>/dev/null; then
# the short wait lets an apps:locked check that briefly takes the lock
# finish instead of failing the deploy
if ! flock -w 1 "$LOCK_FD" &>/dev/null; then
dokku_log_warn "$LOCK_FAILED_MSG"
dokku_log_fail "Run 'apps:unlock' to release the existing deploy lock"
fi
@@ -1054,6 +1078,7 @@ acquire_advisory_lock() {
eval "exec $LOCK_FD>&-"
done
echo "${DOKKU_BUILD_ID:-$DOKKU_PID}" >"$LOCK_FILE"
trap "release_advisory_lock '$LOCK_FILE'" EXIT
}
release_advisory_lock() {
@@ -1061,6 +1086,8 @@ release_advisory_lock() {
local LOCK_FILE="$1"
local LOCK_FD="200"
trap - EXIT
# the file is removed before unlocking so a waiter that locks it afterwards
# can tell that its lock is stale. It is only removed if it is still the file
# this process locked, as apps:unlock may have handed the path to another deploy
+1 -1
View File
@@ -286,7 +286,7 @@ cmd-git-sync() {
# acquire_app_deploy_lock is not used as releasing it would finalize the
# build record before the build starts
acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "exclusive" "$APP currently has a deploy lock in place. Waiting..." "$APP currently has a deploy lock in place. Exiting..."
acquire_app_advisory_lock "$APP" "exclusive"
DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
CURRENT_REF="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null || true)"