From 38ed9bff39457e1b7ae3b3abb02ab08352fb72a6 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 20 Jul 2018 10:37:58 -0400 Subject: [PATCH] feat: add a way to retire old containers scheduled by the docker-local plugin These will be killed on a cron schedule, allowing us to ensure old containers don't run once memory pressure clears up - even across restarts. Closes #3225 --- plugins/00_dokku-standard/install | 2 +- plugins/scheduler-docker-local/install | 37 ++++++++++++++++ .../scheduler-docker-local/internal-functions | 33 ++++++++++++++ .../scheduler-docker-local/scheduler-deploy | 2 + .../scheduler-docker-local/scheduler-retire | 44 +++++++++++++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100755 plugins/scheduler-docker-local/scheduler-retire diff --git a/plugins/00_dokku-standard/install b/plugins/00_dokku-standard/install index e3b199de3..5f708d796 100755 --- a/plugins/00_dokku-standard/install +++ b/plugins/00_dokku-standard/install @@ -9,7 +9,7 @@ if [[ ! -f "$DOKKU_ROOT/VHOST" ]]; then [[ $(dig +short "$(< "$DOKKU_ROOT/HOSTNAME")") ]] && cp "$DOKKU_ROOT/HOSTNAME" "$DOKKU_ROOT/VHOST" fi -dokku_path=$(which dokku) +dokku_path="$(which dokku)" # temporary hack for https://github.com/dokku/dokku/issues/82 # redeploys all apps after a reboot diff --git a/plugins/scheduler-docker-local/install b/plugins/scheduler-docker-local/install index 884c2cb3f..7da527d9e 100755 --- a/plugins/scheduler-docker-local/install +++ b/plugins/scheduler-docker-local/install @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" scheduler-docker-local-install() { @@ -10,6 +11,42 @@ scheduler-docker-local-install() { chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" fn-plugin-property-setup "scheduler-docker-local" + + dokku_path="$(which dokku)" + +if [[ $(systemctl 2> /dev/null) =~ -\.mount ]]; then + cat< /etc/systemd/system/dokku-retire.service +[Unit] +Description=Dokku retire service +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +User=$DOKKU_SYSTEM_USER +ExecStart=$dokku_path ps:restore + +[Install] +WantedBy=docker.service +EOF + + cat< /etc/systemd/system/dokku-retire.timer +[Unit] +Description=Run dokku-retire.service every 2 minutes + +[Timer] +OnCalendar=*:0/2 +EOF + if command -v systemctl &>/dev/null; then + systemctl reenable dokku-redeploy + fi +else + cat< /etc/cron.d/dokku-retire +PATH=/usr/local/bin:/usr/bin:/bin +SHELL=/bin/bash + +2 * * * * $DOKKU_SYSTEM_USER $dokku_path ps:retire +EOF } scheduler-docker-local-install "$@" diff --git a/plugins/scheduler-docker-local/internal-functions b/plugins/scheduler-docker-local/internal-functions index 23648c82e..a63a9826e 100755 --- a/plugins/scheduler-docker-local/internal-functions +++ b/plugins/scheduler-docker-local/internal-functions @@ -87,3 +87,36 @@ cmd-scheduler-docker-local-help() { help_desc fi } + +fn-scheduler-docker-local-retire-container() { + declare APP="$1" CID="$2" DEAD_TIME="$3" + + if ! docker inspect "${CID}" > /dev/null; then + return + fi + + DOKKU_DOCKER_STOP_TIMEOUT="$(config_get "$APP" DOKKU_DOCKER_STOP_TIMEOUT || true)" + [[ $DOKKU_DOCKER_STOP_TIMEOUT ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}" + + # Attempt to stop, if that fails, then force a kill as docker seems + # to not send SIGKILL as the docs would indicate. If that fails, move + # on to the next. + # shellcheck disable=SC2086 + docker stop $DOCKER_STOP_TIME_ARG "$oldid" \ + || docker kill "$oldid" \ + || dokku_log_warn "Unable to kill container ${CID}" + + if ! docker kill "$CID"; then + dokku_log_warn "Unable to kill container ${CID}" + fi +} + +fn-scheduler-docker-local-register-retired-container() { + declare APP="$1" CID="$2" WAIT="$3" + local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers" + local CURRENT_TIME DEAD_TIME + + CURRENT_TIME="$(date +%s)" + DEAD_TIME=$((current_time + $WAIT)) + echo "${APP} ${CID} ${DEAD_TIME}" >> "${DEAD_CONTAINER_FILE}" +} diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index 3faac4875..c586d6cd5 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -4,6 +4,7 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/checks/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" source "$PLUGIN_AVAILABLE_PATH/ps/functions" +source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions" scheduler-docker-local-scheduler-deploy() { declare desc="deploys an image tag for a given application" @@ -155,6 +156,7 @@ scheduler-docker-local-scheduler-deploy() { local oldid for oldid in $oldids; do dokku_log_info2 "$oldid" + fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT" done ( exec >/dev/null 2>/dev/null /dev/null 2>&1; then + dokku_log_warn "Container ${CID} still running" + continue + fi + + docker rm -f "$CID" > /dev/null 2>&1 || true + if docker inspect "${CID}" > /dev/null 2>&1; then + dokku_log_warn "Container ${CID} still running" + continue + fi + + DEAD_CONTAINERS+=("$CID") + done < "$DEAD_CONTAINER_FILE" + + for CID in "${DEAD_CONTAINERS[@]}"; do + sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE" + done +} + +scheduler-docker-local-scheduler-retire "$@"