When a deploy failed, the old containers kept serving while `dokku/<app>:latest` still pointed at the failed image, so `ps:restart`, `ps:start`, and `ps:restore` would redeploy the broken release or fail with `App image not found` once that image was retired. The running image is now kept available during a build and deploy, the `latest` tag is restored to it when the build, release, or deploy fails, and `ps:retire` no longer removes the image an app's `latest` tag points at. Failed containers are also now registered for retirement under the correct app so they and their images are cleaned up.
343 lines
11 KiB
Bash
343 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
|
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
|
|
source "$PLUGIN_AVAILABLE_PATH/config/functions"
|
|
set -eo pipefail
|
|
[[ $DOKKU_TRACE ]] && set -x
|
|
|
|
fn-scheduler-docker-local-init-process() {
|
|
declare APP="$1"
|
|
fn-plugin-property-get-default "scheduler-docker-local" "$APP" "init-process" ""
|
|
}
|
|
|
|
fn-scheduler-docker-local-computed-init-process() {
|
|
declare APP="$1"
|
|
local value
|
|
value="$(fn-scheduler-docker-local-init-process "$APP")"
|
|
if [[ -z "$value" ]]; then
|
|
value="$(fn-scheduler-docker-local-global-init-process "$APP")"
|
|
fi
|
|
if [[ -z "$value" ]]; then
|
|
value="true"
|
|
fi
|
|
echo "$value"
|
|
}
|
|
|
|
fn-scheduler-docker-local-global-init-process() {
|
|
declare APP="$1"
|
|
fn-plugin-property-get-default "scheduler-docker-local" "--global" "init-process" ""
|
|
}
|
|
|
|
fn-scheduler-docker-local-parallel-schedule-count() {
|
|
declare APP="$1"
|
|
fn-plugin-property-get-default "scheduler-docker-local" "$APP" "parallel-schedule-count" ""
|
|
}
|
|
|
|
fn-scheduler-docker-local-computed-parallel-schedule-count() {
|
|
declare APP="$1"
|
|
local value
|
|
value="$(fn-scheduler-docker-local-parallel-schedule-count "$APP")"
|
|
if [[ -z "$value" ]]; then
|
|
value="$(fn-scheduler-docker-local-global-parallel-schedule-count "$APP")"
|
|
fi
|
|
if [[ -z "$value" ]]; then
|
|
value="1"
|
|
fi
|
|
echo "$value"
|
|
}
|
|
|
|
fn-scheduler-docker-local-global-parallel-schedule-count() {
|
|
declare APP="$1"
|
|
fn-plugin-property-get-default "scheduler-docker-local" "--global" "parallel-schedule-count" ""
|
|
}
|
|
|
|
fn-scheduler-docker-local-get-checks-file-path() {
|
|
declare APP="$1"
|
|
|
|
echo "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS"
|
|
}
|
|
|
|
fn-scheduler-docker-local-get-process-specific-checks-file-path() {
|
|
declare APP="$1"
|
|
|
|
checks_path="$(fn-scheduler-docker-local-get-checks-file-path "$APP")"
|
|
process_specific_checks_path="$checks_path.$DOKKU_PID"
|
|
if [[ -f "$process_specific_checks_path" ]]; then
|
|
echo "$process_specific_checks_path"
|
|
return
|
|
fi
|
|
|
|
echo "$checks_path"
|
|
}
|
|
|
|
fn-scheduler-docker-local-has-checks-file() {
|
|
declare APP="$1"
|
|
|
|
checks_path="$(fn-scheduler-docker-local-get-checks-file-path "$APP")"
|
|
if [[ -f "$checks_path.$DOKKU_PID.missing" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
if [[ -f "$checks_path.$DOKKU_PID" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ -f "$checks_path" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
fn-scheduler-docker-local-retire-container() {
|
|
declare APP="$1" CID="$2"
|
|
local STATE
|
|
|
|
dokku_log_verbose_quiet "Attempting to retire $APP container $CID"
|
|
STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
|
|
if [[ -z "$STATE" ]]; then
|
|
return
|
|
fi
|
|
|
|
DOKKU_DOCKER_STOP_TIMEOUT="$(plugn trigger ps-get-property "$APP" stop-timeout-seconds || true)"
|
|
[[ $DOKKU_DOCKER_STOP_TIMEOUT ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}"
|
|
|
|
if [[ "$STATE" == "restarting" ]]; then
|
|
"$DOCKER_BIN" container update --restart=no "$CID" &>/dev/null
|
|
fi
|
|
|
|
if [[ "$STATE" != "dead" ]] && [[ "$STATE" != "exited" ]]; then
|
|
# Attempt to stop, if that fails, then force a kill as docker seems
|
|
# to not send SIGKILL as the docs would indicate. If that fails, move
|
|
# on to the next.
|
|
"$DOCKER_BIN" container stop $DOCKER_STOP_TIME_ARG "$CID" \
|
|
|| "$DOCKER_BIN" container kill "$CID" \
|
|
|| dokku_log_warn "Unable to kill container ${CID}"
|
|
fi
|
|
|
|
STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
|
|
if [[ -z "$STATE" ]]; then
|
|
return
|
|
fi
|
|
|
|
if [[ "$STATE" != "dead" ]] && [[ "$STATE" != "exited" ]]; then
|
|
if ! "$DOCKER_BIN" container kill "$CID"; then
|
|
dokku_log_warn "Unable to kill container ${CID}"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
fn-scheduler-docker-local-retire-containers() {
|
|
local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
|
|
local APP CID CURRENT_TIME DEAD_TIME STATE
|
|
declare SCHEDULER="$1" APP="$2"
|
|
|
|
if [[ ! -f "$DEAD_CONTAINER_FILE" ]]; then
|
|
return
|
|
fi
|
|
|
|
DEAD_CONTAINERS=()
|
|
while read line; do
|
|
[[ -z "$line" ]] && continue
|
|
CURRENT_TIME="$(date +%s)"
|
|
RETIRE_APP="$(echo "$line" | cut -d ' ' -f1)"
|
|
CID="$(echo "$line" | cut -d ' ' -f2)"
|
|
DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
|
|
|
|
if [[ -n "$APP" ]] && [[ "$APP" != "$RETIRE_APP" ]]; then
|
|
continue
|
|
fi
|
|
|
|
if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
|
|
continue
|
|
fi
|
|
|
|
fn-scheduler-docker-local-retire-container "$RETIRE_APP" "$CID"
|
|
STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
|
|
if [[ -z "$STATE" ]]; then
|
|
DEAD_CONTAINERS+=("$CID")
|
|
continue
|
|
fi
|
|
|
|
if [[ "$STATE" == "running" ]]; then
|
|
dokku_log_warn "Container ${CID} still running"
|
|
continue
|
|
fi
|
|
|
|
"$DOCKER_BIN" container rm --force "$CID" &>/dev/null || true
|
|
if "$DOCKER_BIN" container inspect "${CID}" &>/dev/null; then
|
|
dokku_log_warn "Container ${CID} still running"
|
|
continue
|
|
fi
|
|
|
|
DEAD_CONTAINERS+=("$CID")
|
|
done <"$DEAD_CONTAINER_FILE"
|
|
|
|
for CID in "${DEAD_CONTAINERS[@]}"; do
|
|
sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
|
|
done
|
|
}
|
|
|
|
fn-scheduler-docker-local-retire-images() {
|
|
local DEAD_IMAGE_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
|
|
local APP IMAGE_ID CURRENT_TIME DEAD_TIME STATE RM_OUTPUT
|
|
declare SCHEDULER="$1" APP="$2"
|
|
|
|
if [[ ! -f "$DEAD_IMAGE_FILE" ]]; then
|
|
return
|
|
fi
|
|
|
|
local -A SOURCE_IMAGE_APPS=()
|
|
local source_app source_image source_image_id
|
|
for source_app in $(dokku_apps "false" 2>/dev/null); do
|
|
source_image="$(plugn trigger git-get-property "$source_app" "source-image" 2>/dev/null || true)"
|
|
[[ -z "$source_image" ]] && continue
|
|
source_image_id="$("$DOCKER_BIN" image inspect --format '{{ .Id }}' "$source_image" 2>/dev/null | cut -d: -f2 || true)"
|
|
[[ -n "$source_image_id" ]] && SOURCE_IMAGE_APPS["$source_image_id"]="$source_app"
|
|
done
|
|
|
|
DEAD_IMAGES=()
|
|
while read line; do
|
|
[[ -z "$line" ]] && continue
|
|
CURRENT_TIME="$(date +%s)"
|
|
RETIRE_APP="$(echo "$line" | cut -d ' ' -f1)"
|
|
IMAGE_ID="$(echo "$line" | cut -d ' ' -f2)"
|
|
DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
|
|
|
|
if [[ -n "$APP" ]] && [[ "$APP" != "$RETIRE_APP" ]]; then
|
|
continue
|
|
fi
|
|
|
|
if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
|
|
continue
|
|
fi
|
|
|
|
STATE="$("$DOCKER_BIN" image inspect --format "{{ .Id }}" "$IMAGE_ID" 2>/dev/null || true)"
|
|
if [[ -z "$STATE" ]]; then
|
|
DEAD_IMAGES+=("$IMAGE_ID")
|
|
continue
|
|
fi
|
|
|
|
if [[ -n "${SOURCE_IMAGE_APPS[$IMAGE_ID]}" ]]; then
|
|
dokku_log_warn "Image ${IMAGE_ID} is the source image for ${SOURCE_IMAGE_APPS[$IMAGE_ID]}, removing from retire list"
|
|
DEAD_IMAGES+=("$IMAGE_ID")
|
|
continue
|
|
fi
|
|
|
|
if fn-scheduler-docker-local-image-is-app-latest "$RETIRE_APP" "$STATE"; then
|
|
dokku_log_warn "Image ${IMAGE_ID} is the current image for ${RETIRE_APP}, skipping rm"
|
|
continue
|
|
fi
|
|
|
|
dokku_log_verbose_quiet "Attempting to retire $RETIRE_APP image $IMAGE_ID"
|
|
if RM_OUTPUT="$("$DOCKER_BIN" image remove "$IMAGE_ID" 2>&1)"; then
|
|
DEAD_IMAGES+=("$IMAGE_ID")
|
|
continue
|
|
fi
|
|
|
|
if echo "$RM_OUTPUT" | grep -q "image has dependent child images"; then
|
|
TAG_COUNT="$(docker inspect "$IMAGE_ID" --format '{{ json .RepoTags }}' | jq '. | length')"
|
|
if [[ "$TAG_COUNT" -eq 0 ]]; then
|
|
dokku_log_warn "Image ${IMAGE_ID} has children images and is untagged, skipping rm and marking dead"
|
|
DEAD_IMAGES+=("$IMAGE_ID")
|
|
continue
|
|
fi
|
|
|
|
dokku_log_warn "Image ${IMAGE_ID} has children images and has $TAG_COUNT tags, skipping rm"
|
|
continue
|
|
fi
|
|
|
|
if echo "$RM_OUTPUT" | grep -q "image is being used by running container"; then
|
|
if fn-scheduler-docker-local-image-in-use-by-app "$RETIRE_APP" "$IMAGE_ID" ""; then
|
|
dokku_log_warn "Image ${IMAGE_ID} is still in use by ${RETIRE_APP}, removing from retire list"
|
|
DEAD_IMAGES+=("$IMAGE_ID")
|
|
continue
|
|
fi
|
|
dokku_log_warn "Image ${IMAGE_ID} has running containers, skipping rm"
|
|
continue
|
|
fi
|
|
|
|
dokku_log_warn "Image ${IMAGE_ID} still running"
|
|
done <"$DEAD_IMAGE_FILE"
|
|
|
|
for IMAGE_ID in "${DEAD_IMAGES[@]}"; do
|
|
sed -i "/${IMAGE_ID}/d" "$DEAD_IMAGE_FILE"
|
|
done
|
|
|
|
sort -o "$DEAD_IMAGE_FILE" -r "$DEAD_IMAGE_FILE"
|
|
}
|
|
|
|
fn-scheduler-docker-local-image-is-app-latest() {
|
|
declare desc="returns 0 if IMAGE_ID is the image currently tagged as the app's latest image"
|
|
declare APP="$1" IMAGE_ID="$2"
|
|
local LATEST_IMAGE_ID
|
|
|
|
LATEST_IMAGE_ID="$("$DOCKER_BIN" image inspect --format "{{ .Id }}" "$(get_app_image_repo "$APP"):latest" 2>/dev/null || true)"
|
|
[[ -n "$LATEST_IMAGE_ID" ]] && [[ "$LATEST_IMAGE_ID" == "$IMAGE_ID" ]]
|
|
}
|
|
|
|
fn-scheduler-docker-local-register-retired() {
|
|
declare TYPE="$1" APP="$2" DOCKER_ID="$3" WAIT="$4"
|
|
local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
|
|
if [[ "$TYPE" == "image" ]]; then
|
|
local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
|
|
fi
|
|
local CURRENT_TIME DEAD_TIME
|
|
|
|
CURRENT_TIME="$(date +%s)"
|
|
DEAD_TIME=$((CURRENT_TIME + WAIT))
|
|
touch "$DEAD_FILE"
|
|
if ! grep -q "$DOCKER_ID" "$DEAD_FILE"; then
|
|
echo "${APP} ${DOCKER_ID} ${DEAD_TIME}" >>"${DEAD_FILE}"
|
|
fi
|
|
}
|
|
|
|
fn-scheduler-docker-local-image-in-use-by-app() {
|
|
declare desc="returns 0 if IMAGE_ID is used by an app container that is not already pending retirement"
|
|
declare APP="$1" IMAGE_ID="$2" EXCLUDE_CONTAINER_ID="$3"
|
|
local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
|
|
local cid exclude_full_id container_image short_cid
|
|
|
|
if [[ -z "$IMAGE_ID" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
if [[ -n "$EXCLUDE_CONTAINER_ID" ]]; then
|
|
exclude_full_id="$("$DOCKER_BIN" container inspect "$EXCLUDE_CONTAINER_ID" --format '{{.Id}}' 2>/dev/null || true)"
|
|
fi
|
|
|
|
for cid in $("$DOCKER_BIN" container ls -q --no-trunc -f label=com.dokku.app-name="$APP" 2>/dev/null); do
|
|
[[ -z "$cid" ]] && continue
|
|
if [[ -n "$exclude_full_id" ]] && [[ "$cid" == "$exclude_full_id" ]]; then
|
|
continue
|
|
fi
|
|
|
|
short_cid="${cid:0:12}"
|
|
if [[ -f "$DEAD_CONTAINER_FILE" ]] && grep -q "$short_cid" "$DEAD_CONTAINER_FILE"; then
|
|
continue
|
|
fi
|
|
|
|
container_image="$("$DOCKER_BIN" container inspect "$cid" --format '{{.Image}}' 2>/dev/null | cut -d: -f2 || true)"
|
|
if [[ -n "$container_image" ]] && [[ "$container_image" == "$IMAGE_ID" ]]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
fn-scheduler-docker-local-start-app-container() {
|
|
declare desc="starts a single app container"
|
|
declare APP="$1"
|
|
shift
|
|
|
|
declare -a DOCKER_ARGS
|
|
for i in "$@"; do
|
|
DOCKER_ARGS+=("$i")
|
|
done
|
|
set -- "${DOCKER_ARGS[@]}"
|
|
|
|
eval "$(config_export app "$APP" --merged)"
|
|
# shellcheck disable=SC2124
|
|
"$DOCKER_BIN" container create "$@"
|
|
}
|