feat: add a way to retire old containers scheduled by the docker-local plugin

These will be killed on a cron schedule, allowing us to ensure old containers don't run once memory pressure clears up - even across restarts.

Closes #3225

plugins/00_dokku-standard/install

@@ -9,7 +9,7 @@ if [[ ! -f  "$DOKKU_ROOT/VHOST" ]]; then
   [[ $(dig +short "$(< "$DOKKU_ROOT/HOSTNAME")") ]] && cp "$DOKKU_ROOT/HOSTNAME" "$DOKKU_ROOT/VHOST"
 fi
 
-dokku_path=$(which dokku)
+dokku_path="$(which dokku)"
 
 # temporary hack for https://github.com/dokku/dokku/issues/82
 # redeploys all apps after a reboot

plugins/scheduler-docker-local/install

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
 
 scheduler-docker-local-install() {
@@ -10,6 +11,42 @@ scheduler-docker-local-install() {
   chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local"
 
   fn-plugin-property-setup "scheduler-docker-local"
+
+  dokku_path="$(which dokku)"
+
+if [[ $(systemctl 2> /dev/null) =~ -\.mount ]]; then
+  cat<<EOF > /etc/systemd/system/dokku-retire.service
+[Unit]
+Description=Dokku retire service
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+User=$DOKKU_SYSTEM_USER
+ExecStart=$dokku_path ps:restore
+
+[Install]
+WantedBy=docker.service
+EOF
+
+  cat<<EOF > /etc/systemd/system/dokku-retire.timer
+[Unit]
+Description=Run dokku-retire.service every 2 minutes
+
+[Timer]
+OnCalendar=*:0/2
+EOF
+  if command -v systemctl &>/dev/null; then
+    systemctl reenable dokku-redeploy
+  fi
+else
+  cat<<EOF > /etc/cron.d/dokku-retire
+PATH=/usr/local/bin:/usr/bin:/bin
+SHELL=/bin/bash
+
+2 * * * * $DOKKU_SYSTEM_USER $dokku_path ps:retire
+EOF
 }
 
 scheduler-docker-local-install "$@"

plugins/scheduler-docker-local/internal-functions

@@ -87,3 +87,36 @@ cmd-scheduler-docker-local-help() {
 help_desc
   fi
 }
+
+fn-scheduler-docker-local-retire-container() {
+  declare APP="$1" CID="$2" DEAD_TIME="$3"
+
+  if ! docker inspect "${CID}" > /dev/null; then
+    return
+  fi
+
+  DOKKU_DOCKER_STOP_TIMEOUT="$(config_get "$APP" DOKKU_DOCKER_STOP_TIMEOUT || true)"
+  [[ $DOKKU_DOCKER_STOP_TIMEOUT ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}"
+
+  # Attempt to stop, if that fails, then force a kill as docker seems
+  # to not send SIGKILL as the docs would indicate. If that fails, move
+  # on to the next.
+  # shellcheck disable=SC2086
+  docker stop $DOCKER_STOP_TIME_ARG "$oldid" \
+  || docker kill "$oldid" \
+  || dokku_log_warn "Unable to kill container ${CID}"
+
+  if ! docker kill "$CID"; then
+    dokku_log_warn "Unable to kill container ${CID}"
+  fi
+}
+
+fn-scheduler-docker-local-register-retired-container() {
+  declare APP="$1" CID="$2" WAIT="$3"
+  local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers"
+  local CURRENT_TIME DEAD_TIME
+
+  CURRENT_TIME="$(date +%s)"
+  DEAD_TIME=$((current_time + $WAIT))
+  echo "${APP} ${CID} ${DEAD_TIME}" >> "${DEAD_CONTAINER_FILE}"
+}

plugins/scheduler-docker-local/scheduler-deploy

@@ -4,6 +4,7 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 source "$PLUGIN_AVAILABLE_PATH/checks/functions"
 source "$PLUGIN_AVAILABLE_PATH/config/functions"
 source "$PLUGIN_AVAILABLE_PATH/ps/functions"
+source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
 
 scheduler-docker-local-scheduler-deploy() {
   declare desc="deploys an image tag for a given application"
@@ -155,6 +156,7 @@ scheduler-docker-local-scheduler-deploy() {
     local oldid
     for oldid in $oldids; do
       dokku_log_info2 "$oldid"
+      fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT"
     done
     (
       exec >/dev/null 2>/dev/null </dev/null

plugins/scheduler-docker-local/scheduler-retire

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
+
+scheduler-docker-local-scheduler-retire() {
+  declare desc="retires all old containers once they have aged out"
+  declare trigger="scheduler-docker-local scheduler-retire"
+
+  local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers"
+  local APP CID CURRENT_TIME DEAD_TIME
+
+  DEAD_CONTAINERS=()
+  while read line; do
+    CURRENT_TIME="$(date +%s)"
+    APP="$(echo "$line" | cut -d ' ' -f1)"
+    CID="$(echo "$line" | cut -d ' ' -f2)"
+    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
+
+    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
+      continue
+    fi
+
+    fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
+    if docker ps -aq -f "id=${CID}" -f "status=running" > /dev/null 2>&1; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    docker rm -f "$CID" > /dev/null 2>&1 || true
+    if docker inspect "${CID}" > /dev/null 2>&1; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    DEAD_CONTAINERS+=("$CID")
+  done < "$DEAD_CONTAINER_FILE"
+
+  for CID in "${DEAD_CONTAINERS[@]}"; do
+    sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
+  done
+}
+
+scheduler-docker-local-scheduler-retire "$@"