feat: add a way to retire old containers scheduled by the docker-local plugin
These will be killed on a cron schedule, allowing us to ensure old containers don't run once memory pressure clears up - even across restarts. Closes #3225
This commit is contained in:
1 parent
14a1050a26
commit
38ed9bff39
5 files changed
+117
-1
No files matched your search
@@ -9,7 +9,7 @@ if [[ ! -f "$DOKKU_ROOT/VHOST" ]]; then
|
|||||||
[[ $(dig +short "$(< "$DOKKU_ROOT/HOSTNAME")") ]] && cp "$DOKKU_ROOT/HOSTNAME" "$DOKKU_ROOT/VHOST"
|
[[ $(dig +short "$(< "$DOKKU_ROOT/HOSTNAME")") ]] && cp "$DOKKU_ROOT/HOSTNAME" "$DOKKU_ROOT/VHOST"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
dokku_path=$(which dokku)
|
dokku_path="$(which dokku)"
|
||||||
|
|
||||||
# temporary hack for https://github.com/dokku/dokku/issues/82
|
# temporary hack for https://github.com/dokku/dokku/issues/82
|
||||||
# redeploys all apps after a reboot
|
# redeploys all apps after a reboot
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
|
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
|
||||||
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
|
||||||
source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
|
||||||
|
|
||||||
scheduler-docker-local-install() {
|
scheduler-docker-local-install() {
|
||||||
@@ -10,6 +11,42 @@ scheduler-docker-local-install() {
|
|||||||
chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local"
|
chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local"
|
||||||
|
|
||||||
fn-plugin-property-setup "scheduler-docker-local"
|
fn-plugin-property-setup "scheduler-docker-local"
|
||||||
|
|
||||||
|
dokku_path="$(which dokku)"
|
||||||
|
|
||||||
|
if [[ $(systemctl 2> /dev/null) =~ -\.mount ]]; then
|
||||||
|
cat<<EOF > /etc/systemd/system/dokku-retire.service
|
||||||
|
[Unit]
|
||||||
|
Description=Dokku retire service
|
||||||
|
Requires=docker.service
|
||||||
|
After=docker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=$DOKKU_SYSTEM_USER
|
||||||
|
ExecStart=$dokku_path ps:restore
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=docker.service
|
||||||
|
EOF
|
||||||
|
|
||||||
|
cat<<EOF > /etc/systemd/system/dokku-retire.timer
|
||||||
|
[Unit]
|
||||||
|
Description=Run dokku-retire.service every 2 minutes
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*:0/2
|
||||||
|
EOF
|
||||||
|
if command -v systemctl &>/dev/null; then
|
||||||
|
systemctl reenable dokku-redeploy
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
cat<<EOF > /etc/cron.d/dokku-retire
|
||||||
|
PATH=/usr/local/bin:/usr/bin:/bin
|
||||||
|
SHELL=/bin/bash
|
||||||
|
|
||||||
|
2 * * * * $DOKKU_SYSTEM_USER $dokku_path ps:retire
|
||||||
|
EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
scheduler-docker-local-install "$@"
|
scheduler-docker-local-install "$@"
|
||||||
@@ -87,3 +87,36 @@ cmd-scheduler-docker-local-help() {
|
|||||||
help_desc
|
help_desc
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn-scheduler-docker-local-retire-container() {
|
||||||
|
declare APP="$1" CID="$2" DEAD_TIME="$3"
|
||||||
|
|
||||||
|
if ! docker inspect "${CID}" > /dev/null; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOKKU_DOCKER_STOP_TIMEOUT="$(config_get "$APP" DOKKU_DOCKER_STOP_TIMEOUT || true)"
|
||||||
|
[[ $DOKKU_DOCKER_STOP_TIMEOUT ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}"
|
||||||
|
|
||||||
|
# Attempt to stop, if that fails, then force a kill as docker seems
|
||||||
|
# to not send SIGKILL as the docs would indicate. If that fails, move
|
||||||
|
# on to the next.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
docker stop $DOCKER_STOP_TIME_ARG "$oldid" \
|
||||||
|
|| docker kill "$oldid" \
|
||||||
|
|| dokku_log_warn "Unable to kill container ${CID}"
|
||||||
|
|
||||||
|
if ! docker kill "$CID"; then
|
||||||
|
dokku_log_warn "Unable to kill container ${CID}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
fn-scheduler-docker-local-register-retired-container() {
|
||||||
|
declare APP="$1" CID="$2" WAIT="$3"
|
||||||
|
local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers"
|
||||||
|
local CURRENT_TIME DEAD_TIME
|
||||||
|
|
||||||
|
CURRENT_TIME="$(date +%s)"
|
||||||
|
DEAD_TIME=$((current_time + $WAIT))
|
||||||
|
echo "${APP} ${CID} ${DEAD_TIME}" >> "${DEAD_CONTAINER_FILE}"
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
|
|||||||
source "$PLUGIN_AVAILABLE_PATH/checks/functions"
|
source "$PLUGIN_AVAILABLE_PATH/checks/functions"
|
||||||
source "$PLUGIN_AVAILABLE_PATH/config/functions"
|
source "$PLUGIN_AVAILABLE_PATH/config/functions"
|
||||||
source "$PLUGIN_AVAILABLE_PATH/ps/functions"
|
source "$PLUGIN_AVAILABLE_PATH/ps/functions"
|
||||||
|
source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
|
||||||
|
|
||||||
scheduler-docker-local-scheduler-deploy() {
|
scheduler-docker-local-scheduler-deploy() {
|
||||||
declare desc="deploys an image tag for a given application"
|
declare desc="deploys an image tag for a given application"
|
||||||
@@ -155,6 +156,7 @@ scheduler-docker-local-scheduler-deploy() {
|
|||||||
local oldid
|
local oldid
|
||||||
for oldid in $oldids; do
|
for oldid in $oldids; do
|
||||||
dokku_log_info2 "$oldid"
|
dokku_log_info2 "$oldid"
|
||||||
|
fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT"
|
||||||
done
|
done
|
||||||
(
|
(
|
||||||
exec >/dev/null 2>/dev/null </dev/null
|
exec >/dev/null 2>/dev/null </dev/null
|
||||||
|
|||||||
+44
@@ -0,0 +1,44 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
|
||||||
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
|
||||||
|
source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
|
||||||
|
|
||||||
|
scheduler-docker-local-scheduler-retire() {
|
||||||
|
declare desc="retires all old containers once they have aged out"
|
||||||
|
declare trigger="scheduler-docker-local scheduler-retire"
|
||||||
|
|
||||||
|
local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/dokku-scheduler/dead-containers"
|
||||||
|
local APP CID CURRENT_TIME DEAD_TIME
|
||||||
|
|
||||||
|
DEAD_CONTAINERS=()
|
||||||
|
while read line; do
|
||||||
|
CURRENT_TIME="$(date +%s)"
|
||||||
|
APP="$(echo "$line" | cut -d ' ' -f1)"
|
||||||
|
CID="$(echo "$line" | cut -d ' ' -f2)"
|
||||||
|
DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
|
||||||
|
|
||||||
|
if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
|
||||||
|
if docker ps -aq -f "id=${CID}" -f "status=running" > /dev/null 2>&1; then
|
||||||
|
dokku_log_warn "Container ${CID} still running"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker rm -f "$CID" > /dev/null 2>&1 || true
|
||||||
|
if docker inspect "${CID}" > /dev/null 2>&1; then
|
||||||
|
dokku_log_warn "Container ${CID} still running"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
DEAD_CONTAINERS+=("$CID")
|
||||||
|
done < "$DEAD_CONTAINER_FILE"
|
||||||
|
|
||||||
|
for CID in "${DEAD_CONTAINERS[@]}"; do
|
||||||
|
sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduler-docker-local-scheduler-retire "$@"
|
||||||
Reference in new issue
Block a user