fix: release deploy locks on exit and honor apps:lock
A deploy that exited before releasing its lock, such as after a failed build, left `.deploy.lock` behind, so `apps:locked` reported a lock nothing held and the build record stayed running. The lock is now released and the record finalized whenever the deploy exits. `apps:lock` never actually blocked deploys, as it only created the file deploys lock. It now creates its own lock that git pushes, `ps:rebuild`, `git:sync`, and the `git:from-*` commands refuse to deploy past, and `apps:locked` only reports a lock when the app was locked via `apps:lock` or a deploy is in progress. Locks created by `apps:lock` in older versions are migrated on install.
This commit is contained in:
17 files changed
+535
-69
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/dokku/dokku/plugins/common"
|
||||
@@ -19,11 +20,9 @@ func appNameTaken(appName string) bool {
|
||||
return common.DirectoryExists(common.AppRoot(appName))
|
||||
}
|
||||
|
||||
// checks if an app is locked
|
||||
// checks if an app is locked via apps:lock or by an in-progress deploy
|
||||
func appIsLocked(appName string) bool {
|
||||
lockPath := getLockPath(appName)
|
||||
_, err := os.Stat(lockPath)
|
||||
return !os.IsNotExist(err)
|
||||
return appIsManuallyLocked(appName) || deployLockHeld(appName)
|
||||
}
|
||||
|
||||
// verifies app name and creates an app
|
||||
@@ -131,6 +130,38 @@ func getLockPath(appName string) string {
|
||||
return fmt.Sprintf("%v/.deploy.lock", common.GetAppDataDirectory("apps", appName))
|
||||
}
|
||||
|
||||
// returns the path of the lock created by apps:lock
|
||||
func getManualLockPath(appName string) string {
|
||||
return fmt.Sprintf("%v/.manual-deploy.lock", common.GetAppDataDirectory("apps", appName))
|
||||
}
|
||||
|
||||
// checks if an app has been locked via apps:lock
|
||||
func appIsManuallyLocked(appName string) bool {
|
||||
_, err := os.Stat(getManualLockPath(appName))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// checks if a deploy currently holds the flock on the deploy lock file
|
||||
//
|
||||
// a shared lock is attempted and immediately released. Deploys taking the
|
||||
// lock exclusively wait briefly before failing, so this check does not make
|
||||
// a concurrent deploy fail.
|
||||
func deployLockHeld(appName string) bool {
|
||||
lockFile, err := os.Open(getLockPath(appName))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer lockFile.Close()
|
||||
|
||||
fd := int(lockFile.Fd())
|
||||
if err := syscall.Flock(fd, syscall.LOCK_SH|syscall.LOCK_NB); err != nil {
|
||||
return errors.Is(err, syscall.EWOULDBLOCK)
|
||||
}
|
||||
|
||||
_ = syscall.Flock(fd, syscall.LOCK_UN)
|
||||
return false
|
||||
}
|
||||
|
||||
// creates an app if allowed
|
||||
func maybeCreateApp(appName string) error {
|
||||
if err := appExists(appName); err == nil {
|
||||
|
||||
@@ -137,9 +137,7 @@ func CommandLock(appName string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// the file is not truncated as it may hold the build id of an in-progress deploy
|
||||
lockPath := getLockPath(appName)
|
||||
lockFile, err := os.OpenFile(lockPath, os.O_CREATE|os.O_WRONLY, 0644)
|
||||
lockFile, err := os.OpenFile(getManualLockPath(appName), os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return errors.New("Unable to create deploy lock")
|
||||
}
|
||||
@@ -187,6 +185,10 @@ func CommandRename(oldAppName string, newAppName string, skipDeploy bool) error
|
||||
return errors.New("Name is already taken")
|
||||
}
|
||||
|
||||
if appIsManuallyLocked(oldAppName) {
|
||||
return errors.New("Unable to rename a locked app, run apps:unlock first")
|
||||
}
|
||||
|
||||
common.LogInfo1Quiet(fmt.Sprintf("Renaming %s to %s", oldAppName, newAppName))
|
||||
if err := createApp(newAppName); err != nil {
|
||||
return err
|
||||
@@ -264,13 +266,12 @@ func CommandUnlock(appName string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
lockfilePath := getLockPath(appName)
|
||||
if _, err := os.Stat(lockfilePath); !os.IsNotExist(err) {
|
||||
if deployLockHeld(appName) {
|
||||
common.LogWarn("A deploy may be in progress.")
|
||||
common.LogWarn("Removing the app lock will not stop in progress deploys.")
|
||||
}
|
||||
|
||||
if err := os.Remove(lockfilePath); err != nil {
|
||||
if err := removeLockFiles(getManualLockPath(appName), getLockPath(appName)); err != nil {
|
||||
return errors.New("Unable to remove deploy lock")
|
||||
}
|
||||
|
||||
|
||||
@@ -83,6 +83,40 @@ func TriggerInstall() error {
|
||||
}
|
||||
}
|
||||
|
||||
for _, appName := range apps {
|
||||
if err := migrateManualLock(appName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateManualLock converts a lock created by apps:lock before manual locks
|
||||
// had their own file. Deploys write their build id into the deploy lock file,
|
||||
// so an empty one that no deploy holds was created by apps:lock.
|
||||
func migrateManualLock(appName string) error {
|
||||
lockPath := getLockPath(appName)
|
||||
fi, err := os.Stat(lockPath)
|
||||
if err != nil || fi.Size() != 0 || deployLockHeld(appName) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.Rename(lockPath, getManualLockPath(appName)); err != nil {
|
||||
return fmt.Errorf("Unable to migrate the deploy lock for %s: %w", appName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeLockFiles removes the given lock files, ignoring ones that do not exist
|
||||
func removeLockFiles(lockPaths ...string) error {
|
||||
for _, lockPath := range lockPaths {
|
||||
if err := os.Remove(lockPath); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -93,7 +127,11 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return common.CloneAppData("apps", oldAppName, newAppName)
|
||||
if err := common.CloneAppData("apps", oldAppName, newAppName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return removeLockFiles(getManualLockPath(newAppName), getLockPath(newAppName))
|
||||
}
|
||||
|
||||
// TriggerPostAppRename removes the old app data
|
||||
@@ -111,7 +149,11 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return common.CloneAppData("apps", oldAppName, newAppName)
|
||||
if err := common.CloneAppData("apps", oldAppName, newAppName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return removeLockFiles(getLockPath(newAppName))
|
||||
}
|
||||
|
||||
// TriggerPostCreate ensures apps have the correct data directory structure
|
||||
|
||||
@@ -319,15 +319,12 @@ func CommandCancel(appName string) error {
|
||||
if err := WriteBuild(b); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(lockPath)
|
||||
removeLockForBuild(lockPath, buildID)
|
||||
return nil
|
||||
}
|
||||
|
||||
common.LogInfo1("Killing app deploy")
|
||||
if err := killProcessGroup(b.PID); err != nil {
|
||||
common.LogWarn(fmt.Sprintf("Failed to signal process group %d: %s", b.PID, err))
|
||||
}
|
||||
|
||||
// the record is finalized before the signal is sent so that a deploy
|
||||
// finalizing itself on exit leaves it as canceled
|
||||
current, err := ReadBuild(appName, buildID)
|
||||
if err == nil && current.Status == BuildStatusRunning {
|
||||
now := time.Now().UTC()
|
||||
@@ -340,10 +337,30 @@ func CommandCancel(appName string) error {
|
||||
}
|
||||
}
|
||||
|
||||
_ = os.Remove(lockPath)
|
||||
common.LogInfo1("Killing app deploy")
|
||||
if err := killProcessGroup(b.PID); err != nil {
|
||||
common.LogWarn(fmt.Sprintf("Failed to signal process group %d: %s", b.PID, err))
|
||||
}
|
||||
|
||||
removeLockForBuild(lockPath, buildID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeLockForBuild removes the deploy lock file if it still holds the given
|
||||
// build id, leaving a lock taken by a newer deploy in place
|
||||
func removeLockForBuild(lockPath string, buildID string) {
|
||||
body, err := os.ReadFile(lockPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if strings.TrimSpace(string(body)) != buildID {
|
||||
return
|
||||
}
|
||||
|
||||
_ = os.Remove(lockPath)
|
||||
}
|
||||
|
||||
func killProcessGroup(pid int) error {
|
||||
pgid, err := syscall.Getpgid(pid)
|
||||
if err != nil {
|
||||
|
||||
@@ -986,14 +986,35 @@ app_deploy_lock_file() {
|
||||
echo "$DOKKU_LIB_ROOT/data/apps/$APP/.deploy.lock"
|
||||
}
|
||||
|
||||
acquire_app_deploy_lock() {
|
||||
declare desc="acquire advisory lock for use in deploys"
|
||||
local APP="$1"
|
||||
local LOCK_TYPE="${2:-waiting}"
|
||||
app_manual_lock_file() {
|
||||
declare desc="outputs the path to the manual lock file created by apps:lock for an app"
|
||||
declare APP="$1"
|
||||
echo "$DOKKU_LIB_ROOT/data/apps/$APP/.manual-deploy.lock"
|
||||
}
|
||||
|
||||
acquire_app_advisory_lock() {
|
||||
declare desc="acquire the advisory lock for an app's deploys, failing if the app was locked via apps:lock"
|
||||
declare APP="$1" LOCK_TYPE="${2:-waiting}"
|
||||
local APP_DEPLOY_LOCK_FILE="$(app_deploy_lock_file "$APP")"
|
||||
local LOCK_WAITING_MSG="$APP currently has a deploy lock in place. Waiting..."
|
||||
local LOCK_FAILED_MSG="$APP currently has a deploy lock in place. Exiting..."
|
||||
|
||||
acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG"
|
||||
if [[ ! -f "$(app_manual_lock_file "$APP")" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
release_advisory_lock "$APP_DEPLOY_LOCK_FILE"
|
||||
dokku_discard_build_capture "$APP"
|
||||
dokku_log_warn "$APP has been locked via apps:lock"
|
||||
dokku_log_fail "Run 'apps:unlock' to allow deploys"
|
||||
}
|
||||
|
||||
acquire_app_deploy_lock() {
|
||||
declare desc="acquire advisory lock for use in deploys, releasing it and finalizing the build record when the shell exits"
|
||||
local APP="$1"
|
||||
local LOCK_TYPE="${2:-waiting}"
|
||||
|
||||
if [[ -z "$DOKKU_BUILD_ID" ]]; then
|
||||
DOKKU_BUILD_ID="$(plugn trigger builds-generate-id 2>/dev/null)"
|
||||
if [[ -z "$DOKKU_BUILD_ID" ]]; then
|
||||
@@ -1002,7 +1023,8 @@ acquire_app_deploy_lock() {
|
||||
export DOKKU_BUILD_ID
|
||||
fi
|
||||
|
||||
acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG"
|
||||
acquire_app_advisory_lock "$APP" "$LOCK_TYPE"
|
||||
trap "release_app_deploy_lock '$APP' \"\$?\"" EXIT
|
||||
}
|
||||
|
||||
release_app_deploy_lock() {
|
||||
@@ -1040,7 +1062,9 @@ acquire_advisory_lock() {
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
if ! flock -n "$LOCK_FD" &>/dev/null; then
|
||||
# the short wait lets an apps:locked check that briefly takes the lock
|
||||
# finish instead of failing the deploy
|
||||
if ! flock -w 1 "$LOCK_FD" &>/dev/null; then
|
||||
dokku_log_warn "$LOCK_FAILED_MSG"
|
||||
dokku_log_fail "Run 'apps:unlock' to release the existing deploy lock"
|
||||
fi
|
||||
@@ -1054,6 +1078,7 @@ acquire_advisory_lock() {
|
||||
eval "exec $LOCK_FD>&-"
|
||||
done
|
||||
echo "${DOKKU_BUILD_ID:-$DOKKU_PID}" >"$LOCK_FILE"
|
||||
trap "release_advisory_lock '$LOCK_FILE'" EXIT
|
||||
}
|
||||
|
||||
release_advisory_lock() {
|
||||
@@ -1061,6 +1086,8 @@ release_advisory_lock() {
|
||||
local LOCK_FILE="$1"
|
||||
local LOCK_FD="200"
|
||||
|
||||
trap - EXIT
|
||||
|
||||
# the file is removed before unlocking so a waiter that locks it afterwards
|
||||
# can tell that its lock is stale. It is only removed if it is still the file
|
||||
# this process locked, as apps:unlock may have handed the path to another deploy
|
||||
|
||||
@@ -286,7 +286,7 @@ cmd-git-sync() {
|
||||
|
||||
# acquire_app_deploy_lock is not used as releasing it would finalize the
|
||||
# build record before the build starts
|
||||
acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "exclusive" "$APP currently has a deploy lock in place. Waiting..." "$APP currently has a deploy lock in place. Exiting..."
|
||||
acquire_app_advisory_lock "$APP" "exclusive"
|
||||
|
||||
DOKKU_DEPLOY_BRANCH="$(fn-git-computed-deploy-branch "$APP")"
|
||||
CURRENT_REF="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null || true)"
|
||||
|
||||
Reference in new issue
Block a user