From 48842fc2f62d3e0bea87beb6abaaeb48b6dbd8fd Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 20 Jul 2018 02:53:17 -0400 Subject: [PATCH 1/9] refactor: allow any function to attain a lock --- plugins/common/functions | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/plugins/common/functions b/plugins/common/functions index fa0d63835..ebeeab7f4 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -943,15 +943,30 @@ acquire_app_deploy_lock() { declare desc="acquire advisory lock for use in git/tar deploys" local APP="$1"; verify_app_name "$APP" local LOCK_TYPE="${2:-waiting}" - local APP_DEPLOY_LOCK_FD="200" local APP_DEPLOY_LOCK_FILE="$DOKKU_ROOT/$APP/.deploy.lock" local LOCK_WAITING_MSG="$APP currently has a deploy lock in place. Waiting..." local LOCK_FAILED_MSG="$APP currently has a deploy lock in place. Exiting..." + + acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG" +} + +release_app_deploy_lock() { + declare desc="release advisory lock used in git/tar deploys" + local APP="$1"; verify_app_name "$APP" + local APP_DEPLOY_LOCK_FILE="$DOKKU_ROOT/$APP/.deploy.lock" + + release_advisory_lock "$APP_DEPLOY_LOCK_FILE" +} + +acquire_advisory_lock() { + declare desc="acquire advisory lock" + local LOCK_FILE="$1" LOCK_TYPE="$2" LOCK_WAITING_MSG="$3" LOCK_FAILED_MSG="$4" + local LOCK_FD="200" local SHOW_MSG=true - eval "exec $APP_DEPLOY_LOCK_FD>$APP_DEPLOY_LOCK_FILE" + eval "exec $LOCK_FD>$APP_DEPLOY_LOCK_FILE" if [[ "$LOCK_TYPE" == "waiting" ]]; then - while [[ $(flock -n "$APP_DEPLOY_LOCK_FD" &>/dev/null ; echo $?) -ne 0 ]]; do + while [[ $(flock -n "$LOCK_FD" &>/dev/null ; echo $?) -ne 0 ]]; do if [[ "$SHOW_MSG" == "true" ]]; then echo "$LOCK_WAITING_MSG" SHOW_MSG=false @@ -959,17 +974,16 @@ acquire_app_deploy_lock() { sleep 1 done else - flock -n "$APP_DEPLOY_LOCK_FD" &>/dev/null || dokku_log_fail "$LOCK_FAILED_MSG" + flock -n "$LOCK_FD" &>/dev/null || dokku_log_fail "$LOCK_FAILED_MSG" fi } -release_app_deploy_lock() { - declare desc="release advisory lock used in git/tar deploys" - local APP="$1"; verify_app_name "$APP" - local APP_DEPLOY_LOCK_FD="200" - local APP_DEPLOY_LOCK_FILE="$DOKKU_ROOT/$APP/.deploy.lock" +release_advisory_lock() { + declare desc="release advisory lock" + local LOCK_FILE="$1" + local LOCK_FD="200" - flock -u "$APP_DEPLOY_LOCK_FD" && rm -f "$APP_DEPLOY_LOCK_FILE" &> /dev/null + flock -u "$LOCK_FD" && rm -f "$LOCK_FILE" &> /dev/null } suppress_output() { From 14a1050a261d9d5a7907093912afa1b53bada108 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 20 Jul 2018 10:05:04 -0400 Subject: [PATCH 2/9] feat: add ps:retire command This can be used to remove all old containers for a given deployment method. --- plugins/ps/subcommands/retire | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 plugins/ps/subcommands/retire diff --git a/plugins/ps/subcommands/retire b/plugins/ps/subcommands/retire new file mode 100755 index 000000000..221f0d551 --- /dev/null +++ b/plugins/ps/subcommands/retire @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_AVAILABLE_PATH/config/functions" +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" + +ps_retire_cmd() { + declare desc="ensures old containers are retired" + local cmd="ps:retire" + local LOCK_FILE="${DOKKU_LIB_ROOT}/data/ps/retire" + local DOKKU_SCHEDULER=$(config_get "--global" DOKKU_SCHEDULER || echo "docker-local") + + acquire_advisory_lock "$LOCK_FILE" "exclusive" "" "Failed to acquire ps:retire lock" + plugn trigger scheduler-retire "$DOKKU_SCHEDULER" + release_advisory_lock "$LOCK_FILE" +} + +ps_retire_cmd "$@" From 38ed9bff39457e1b7ae3b3abb02ab08352fb72a6 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 20 Jul 2018 10:37:58 -0400 Subject: [PATCH 3/9] feat: add a way to retire old containers scheduled by the docker-local plugin These will be killed on a cron schedule, allowing us to ensure old containers don't run once memory pressure clears up - even across restarts. Closes #3225 --- plugins/00_dokku-standard/install | 2 +- plugins/scheduler-docker-local/install | 37 ++++++++++++++++ .../scheduler-docker-local/internal-functions | 33 ++++++++++++++ .../scheduler-docker-local/scheduler-deploy | 2 + .../scheduler-docker-local/scheduler-retire | 44 +++++++++++++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100755 plugins/scheduler-docker-local/scheduler-retire diff --git a/plugins/00_dokku-standard/install b/plugins/00_dokku-standard/install index e3b199de3..5f708d796 100755 --- a/plugins/00_dokku-standard/install +++ b/plugins/00_dokku-standard/install @@ -9,7 +9,7 @@ if [[ ! -f "$DOKKU_ROOT/VHOST" ]]; then [[ $(dig +short "$(< "$DOKKU_ROOT/HOSTNAME")") ]] && cp "$DOKKU_ROOT/HOSTNAME" "$DOKKU_ROOT/VHOST" fi -dokku_path=$(which dokku) +dokku_path="$(which dokku)" # temporary hack for https://github.com/dokku/dokku/issues/82 # redeploys all apps after a reboot diff --git a/plugins/scheduler-docker-local/install b/plugins/scheduler-docker-local/install index 884c2cb3f..7da527d9e 100755 --- a/plugins/scheduler-docker-local/install +++ b/plugins/scheduler-docker-local/install @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" scheduler-docker-local-install() { @@ -10,6 +11,42 @@ scheduler-docker-local-install() { chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" fn-plugin-property-setup "scheduler-docker-local" + + dokku_path="$(which dokku)" + +if [[ $(systemctl 2> /dev/null) =~ -\.mount ]]; then + cat< /etc/systemd/system/dokku-retire.service +[Unit] +Description=Dokku retire service +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +User=$DOKKU_SYSTEM_USER +ExecStart=$dokku_path ps:restore + +[Install] +WantedBy=docker.service +EOF + + cat< /etc/systemd/system/dokku-retire.timer +[Unit] +Description=Run dokku-retire.service every 2 minutes + +[Timer] +OnCalendar=*:0/2 +EOF + if command -v systemctl &>/dev/null; then + systemctl reenable dokku-redeploy + fi +else + cat< /etc/cron.d/dokku-retire +PATH=/usr/local/bin:/usr/bin:/bin +SHELL=/bin/bash + +2 * * * * $DOKKU_SYSTEM_USER $dokku_path ps:retire +EOF } scheduler-docker-local-install "$@" diff --git a/plugins/scheduler-docker-local/internal-functions b/plugins/scheduler-docker-local/internal-functions index 23648c82e..a63a9826e 100755 --- a/plugins/scheduler-docker-local/internal-functions +++ b/plugins/scheduler-docker-local/internal-functions @@ -87,3 +87,36 @@ cmd-scheduler-docker-local-help() { help_desc fi } + +fn-scheduler-docker-local-retire-container() { + declare APP="$1" CID="$2" DEAD_TIME="$3" + + if ! docker inspect "${CID}" > /dev/null; then + return + fi + + DOKKU_DOCKER_STOP_TIMEOUT="$(config_get "$APP" DOKKU_DOCKER_STOP_TIMEOUT || true)" + [[ $DOKKU_DOCKER_STOP_TIMEOUT ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}" + + # Attempt to stop, if that fails, then force a kill as docker seems + # to not send SIGKILL as the docs would indicate. If that fails, move + # on to the next. + # shellcheck disable=SC2086 + docker stop $DOCKER_STOP_TIME_ARG "$oldid" \ + || docker kill "$oldid" \ + || dokku_log_warn "Unable to kill container ${CID}" + + if ! docker kill "$CID"; then + dokku_log_warn "Unable to kill container ${CID}" + fi +} + +fn-scheduler-docker-local-register-retired-container() { + declare APP="$1" CID="$2" WAIT="$3" + local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers" + local CURRENT_TIME DEAD_TIME + + CURRENT_TIME="$(date +%s)" + DEAD_TIME=$((current_time + $WAIT)) + echo "${APP} ${CID} ${DEAD_TIME}" >> "${DEAD_CONTAINER_FILE}" +} diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index 3faac4875..c586d6cd5 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -4,6 +4,7 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/checks/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" source "$PLUGIN_AVAILABLE_PATH/ps/functions" +source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions" scheduler-docker-local-scheduler-deploy() { declare desc="deploys an image tag for a given application" @@ -155,6 +156,7 @@ scheduler-docker-local-scheduler-deploy() { local oldid for oldid in $oldids; do dokku_log_info2 "$oldid" + fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT" done ( exec >/dev/null 2>/dev/null /dev/null 2>&1; then + dokku_log_warn "Container ${CID} still running" + continue + fi + + docker rm -f "$CID" > /dev/null 2>&1 || true + if docker inspect "${CID}" > /dev/null 2>&1; then + dokku_log_warn "Container ${CID} still running" + continue + fi + + DEAD_CONTAINERS+=("$CID") + done < "$DEAD_CONTAINER_FILE" + + for CID in "${DEAD_CONTAINERS[@]}"; do + sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE" + done +} + +scheduler-docker-local-scheduler-retire "$@" From d9fca6ac476608d69786d889cf1d8a35f71037ff Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 20 Jul 2018 12:28:35 -0400 Subject: [PATCH 4/9] fix: correct syntax error --- plugins/scheduler-docker-local/install | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/scheduler-docker-local/install b/plugins/scheduler-docker-local/install index 7da527d9e..faf0b5001 100755 --- a/plugins/scheduler-docker-local/install +++ b/plugins/scheduler-docker-local/install @@ -47,6 +47,7 @@ SHELL=/bin/bash 2 * * * * $DOKKU_SYSTEM_USER $dokku_path ps:retire EOF +fi } scheduler-docker-local-install "$@" From 8a9c77a061dcb191f6c83cde8837bc3dd5f41353 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 20 Jul 2018 13:08:15 -0400 Subject: [PATCH 5/9] fix: pass lint --- plugins/scheduler-docker-local/internal-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scheduler-docker-local/internal-functions b/plugins/scheduler-docker-local/internal-functions index a63a9826e..93654347b 100755 --- a/plugins/scheduler-docker-local/internal-functions +++ b/plugins/scheduler-docker-local/internal-functions @@ -117,6 +117,6 @@ fn-scheduler-docker-local-register-retired-container() { local CURRENT_TIME DEAD_TIME CURRENT_TIME="$(date +%s)" - DEAD_TIME=$((current_time + $WAIT)) + DEAD_TIME=$((CURRENT_TIME + WAIT)) echo "${APP} ${CID} ${DEAD_TIME}" >> "${DEAD_CONTAINER_FILE}" } From deeaf6dc4181a6a0b48f63a596d298db1ad536c8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 20 Jul 2018 14:29:00 -0400 Subject: [PATCH 6/9] fix: ensure path exists --- plugins/scheduler-docker-local/install | 7 ++++--- plugins/scheduler-docker-local/internal-functions | 2 +- plugins/scheduler-docker-local/scheduler-retire | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/plugins/scheduler-docker-local/install b/plugins/scheduler-docker-local/install index faf0b5001..230cf687a 100755 --- a/plugins/scheduler-docker-local/install +++ b/plugins/scheduler-docker-local/install @@ -6,13 +6,14 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" scheduler-docker-local-install() { declare desc="scheduler-docker-local install plugin trigger" declare trigger="scheduler-docker-local-install" + local DOKKU_PATH mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" fn-plugin-property-setup "scheduler-docker-local" - dokku_path="$(which dokku)" + DOKKU_PATH="$(which dokku)" if [[ $(systemctl 2> /dev/null) =~ -\.mount ]]; then cat< /etc/systemd/system/dokku-retire.service @@ -24,7 +25,7 @@ After=docker.service [Service] Type=oneshot User=$DOKKU_SYSTEM_USER -ExecStart=$dokku_path ps:restore +ExecStart=$DOKKU_PATH ps:restore [Install] WantedBy=docker.service @@ -45,7 +46,7 @@ else PATH=/usr/local/bin:/usr/bin:/bin SHELL=/bin/bash -2 * * * * $DOKKU_SYSTEM_USER $dokku_path ps:retire +2 * * * * $DOKKU_SYSTEM_USER $DOKKU_PATH ps:retire EOF fi } diff --git a/plugins/scheduler-docker-local/internal-functions b/plugins/scheduler-docker-local/internal-functions index 93654347b..338441705 100755 --- a/plugins/scheduler-docker-local/internal-functions +++ b/plugins/scheduler-docker-local/internal-functions @@ -113,7 +113,7 @@ fn-scheduler-docker-local-retire-container() { fn-scheduler-docker-local-register-retired-container() { declare APP="$1" CID="$2" WAIT="$3" - local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers" + local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers" local CURRENT_TIME DEAD_TIME CURRENT_TIME="$(date +%s)" diff --git a/plugins/scheduler-docker-local/scheduler-retire b/plugins/scheduler-docker-local/scheduler-retire index 92df8797d..f0f24cd02 100755 --- a/plugins/scheduler-docker-local/scheduler-retire +++ b/plugins/scheduler-docker-local/scheduler-retire @@ -7,7 +7,7 @@ scheduler-docker-local-scheduler-retire() { declare desc="retires all old containers once they have aged out" declare trigger="scheduler-docker-local scheduler-retire" - local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers" + local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers" local APP CID CURRENT_TIME DEAD_TIME DEAD_CONTAINERS=() From ec47b1bcc7cb75b8e4d8f8dac2332a8c9a505d7d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Jul 2018 13:20:16 -0400 Subject: [PATCH 7/9] refactor: set DOKKU_SYSTEM_USER and DOKKU_SYSTEM_GROUP at the top-level --- plugins/common/properties.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/plugins/common/properties.go b/plugins/common/properties.go index 61da772f3..46977b983 100644 --- a/plugins/common/properties.go +++ b/plugins/common/properties.go @@ -144,13 +144,6 @@ func setPermissions(path string, fileMode os.FileMode) (err error) { systemGroup := os.Getenv("DOKKU_SYSTEM_GROUP") systemUser := os.Getenv("DOKKU_SYSTEM_USER") - if systemGroup == "" { - systemGroup = "dokku" - } - if systemUser == "" { - systemUser = "dokku" - } - group, err := user.LookupGroup(systemGroup) if err != nil { return From c1b47dd248fe9fa2b7df8176d953ea739a62ead6 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 26 Jul 2018 10:09:02 -0400 Subject: [PATCH 8/9] fix: use correct path for lockfile --- plugins/common/functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/common/functions b/plugins/common/functions index ebeeab7f4..75694ac82 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -964,7 +964,7 @@ acquire_advisory_lock() { local LOCK_FD="200" local SHOW_MSG=true - eval "exec $LOCK_FD>$APP_DEPLOY_LOCK_FILE" + eval "exec $LOCK_FD>$LOCK_FILE" if [[ "$LOCK_TYPE" == "waiting" ]]; then while [[ $(flock -n "$LOCK_FD" &>/dev/null ; echo $?) -ne 0 ]]; do if [[ "$SHOW_MSG" == "true" ]]; then From f83aba565f037992887462d593a58e78a5da4b08 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 26 Jul 2018 10:09:18 -0400 Subject: [PATCH 9/9] fix: properly chown ps folder --- plugins/ps/install | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/ps/install b/plugins/ps/install index 51195449d..b70187689 100755 --- a/plugins/ps/install +++ b/plugins/ps/install @@ -4,11 +4,14 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/docker-options/functions" source "$PLUGIN_AVAILABLE_PATH/ps/functions" -set_default_restart_policies() { +trigger-ps-install() { declare desc="set the default restart policy for all applications if there is not one already set" local APPS="$(dokku_apps)" local APP + mkdir -p "${DOKKU_LIB_ROOT}/data/ps" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/ps" + for APP in $APPS; do local RESTART_POLICIES=$(get_restart_policies "$(get_phase_file_path "deploy")") if [[ -z "$RESTART_POLICIES" ]]; then @@ -18,4 +21,4 @@ set_default_restart_policies() { done } -set_default_restart_policies "$@" +trigger-ps-install "$@"