feat: schedule related images for cleanup when retiring containers

If the containers no longer exist and the images are unused, we should remove the images as well.

Note that images may sometimes stick around for a little while, depending on the retire schedule and the wait time specified for the container.

Closes #4104

docs/development/plugin-triggers.md

@@ -1914,7 +1914,7 @@ DOKKU_SCHEDULER="$1"; APP="$2"; CONTAINER_ID="$3";
 > Warning: The scheduler plugin trigger apis are under development and may change
 > between minor releases until the 1.0 release.
 
-- Description: Allows scheduling retiring a local container
+- Description: Allows scheduling retiring a local container and any related images
 - Invoked by: `internally`
 - Arguments: `$APP $CONTAINER_ID`
 - Example:

plugins/scheduler-docker-local/internal-functions

@@ -105,12 +105,116 @@ fn-scheduler-docker-local-retire-container() {
   fi
 }
 
-fn-scheduler-docker-local-register-retired-container() {
-  declare APP="$1" CID="$2" WAIT="$3"
+fn-scheduler-docker-local-retire-containers() {
   local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
+  local APP CID CURRENT_TIME DEAD_TIME STATE
+
+  if [[ ! -f "$DEAD_CONTAINER_FILE" ]]; then
+    return
+  fi
+
+  DEAD_CONTAINERS=()
+  while read line; do
+    [[ -z "$line" ]] && continue
+    CURRENT_TIME="$(date +%s)"
+    APP="$(echo "$line" | cut -d ' ' -f1)"
+    CID="$(echo "$line" | cut -d ' ' -f2)"
+    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
+
+    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
+      continue
+    fi
+
+    fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
+    STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
+    if [[ -z "$STATE" ]]; then
+      DEAD_CONTAINERS+=("$CID")
+      continue
+    fi
+
+    if [[ "$STATE" == "running" ]]; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    "$DOCKER_BIN" container rm --force "$CID" >/dev/null 2>&1 || true
+    if "$DOCKER_BIN" container inspect "${CID}" >/dev/null 2>&1; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    DEAD_CONTAINERS+=("$CID")
+  done <"$DEAD_CONTAINER_FILE"
+
+  for CID in "${DEAD_CONTAINERS[@]}"; do
+    sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
+  done
+}
+
+fn-scheduler-docker-local-retire-images() {
+  local DEAD_IMAGE_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
+  local APP IMAGE_ID CURRENT_TIME DEAD_TIME STATE RM_OUTPUT
+
+  if [[ ! -f "$DEAD_IMAGE_FILE" ]]; then
+    return
+  fi
+
+  DEAD_IMAGES=()
+  while read line; do
+    [[ -z "$line" ]] && continue
+    CURRENT_TIME="$(date +%s)"
+    APP="$(echo "$line" | cut -d ' ' -f1)"
+    IMAGE_ID="$(echo "$line" | cut -d ' ' -f2)"
+    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
+
+    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
+      continue
+    fi
+
+    STATE="$("$DOCKER_BIN" image inspect --format "{{ .Id }}" "$IMAGE_ID" 2>/dev/null || true)"
+    if [[ -z "$STATE" ]]; then
+      DEAD_IMAGES+=("$IMAGE_ID")
+      continue
+    fi
+
+    dokku_log_info1_quiet "Attempting to retire $APP image $IMAGE_ID"
+    if RM_OUTPUT="$("$DOCKER_BIN" image rm "$IMAGE_ID" 2>&1)"; then
+      DEAD_IMAGES+=("$IMAGE_ID")
+      continue
+    fi
+
+    if echo "$RM_OUTPUT" | grep -q "image has dependent child images"; then
+      dokku_log_warn "Image ${IMAGE_ID} has children images, skipping rm and marking dead"
+      DEAD_IMAGES+=("$IMAGE_ID")
+      continue
+    fi
+
+    if echo "$RM_OUTPUT" | grep -q "image is being used by running container"; then
+      dokku_log_warn "Image ${IMAGE_ID} has running containers, skipping rm"
+      continue
+    fi
+
+    dokku_log_warn "Image ${IMAGE_ID} still running"
+  done <"$DEAD_IMAGE_FILE"
+
+  for IMAGE_ID in "${DEAD_IMAGES[@]}"; do
+    sed -i "/${IMAGE_ID}/d" "$DEAD_IMAGE_FILE"
+  done
+
+  sort -o "$DEAD_IMAGE_FILE" -r "$DEAD_IMAGE_FILE"
+}
+
+fn-scheduler-docker-local-register-retired() {
+  declare TYPE="$1" APP="$2" DOCKER_ID="$3" WAIT="$4"
+  local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
+  if [[ "$TYPE" == "image" ]]; then
+    local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
+  fi
   local CURRENT_TIME DEAD_TIME
 
   CURRENT_TIME="$(date +%s)"
   DEAD_TIME=$((CURRENT_TIME + WAIT))
-  echo "${APP} ${CID} ${DEAD_TIME}" >>"${DEAD_CONTAINER_FILE}"
+  if ! grep -q "$DOCKER_ID" "$DEAD_FILE"; then
+    echo "${APP} ${DOCKER_ID} ${DEAD_TIME}" >>"${DEAD_FILE}"
+  fi
 }

plugins/scheduler-docker-local/scheduler-deploy

@@ -180,7 +180,7 @@ trigger-scheduler-docker-local-scheduler-deploy() {
     local oldid
     for oldid in $oldids; do
       dokku_log_verbose_quiet "$oldid"
-      fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT"
+      plugn trigger scheduler-register-retired "$APP" "$oldid" "$WAIT"
     done
     (
       exec >/dev/null 2>/dev/null </dev/null

plugins/scheduler-docker-local/scheduler-register-retired

@@ -7,8 +7,12 @@ trigger-scheduler-docker-local-scheduler-register-retired() {
   declare desc="register a container for retiring"
   declare trigger="scheduler-register-retired"
   declare APP="$1" CONTAINER_ID="$2" WAIT="${3:-60}"
+  local IMAGE_ID
+
+  IMAGE_ID="$("$DOCKER_BIN" container inspect "$CONTAINER_ID" --format "{{.Image}}" | cut -d: -f2)"
+  fn-scheduler-docker-local-register-retired "image" "$APP" "$IMAGE_ID" "$WAIT"
+  fn-scheduler-docker-local-register-retired "container" "$APP" "$CONTAINER_ID" "$WAIT"
 
-  fn-scheduler-docker-local-register-retired-container "$APP" "$CONTAINER_ID" "$WAIT"
 }
 
 trigger-scheduler-docker-local-scheduler-register-retired "$@"

plugins/scheduler-docker-local/scheduler-retire

@@ -8,48 +8,8 @@ trigger-scheduler-docker-local-scheduler-retire() {
   declare desc="retires all old containers once they have aged out"
   declare trigger="scheduler-retire"
 
-  local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
-  local APP CID CURRENT_TIME DEAD_TIME STATE
-
-  if [[ ! -f "$DEAD_CONTAINER_FILE" ]]; then
-    return
-  fi
-
-  DEAD_CONTAINERS=()
-  while read line; do
-    CURRENT_TIME="$(date +%s)"
-    APP="$(echo "$line" | cut -d ' ' -f1)"
-    CID="$(echo "$line" | cut -d ' ' -f2)"
-    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
-
-    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
-      continue
-    fi
-
-    fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
-    STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
-    if [[ -z "$STATE" ]]; then
-      DEAD_CONTAINERS+=("$CID")
-      continue
-    fi
-
-    if [[ "$STATE" == "running" ]]; then
-      dokku_log_warn "Container ${CID} still running"
-      continue
-    fi
-
-    "$DOCKER_BIN" container rm --force "$CID" >/dev/null 2>&1 || true
-    if "$DOCKER_BIN" container inspect "${CID}" >/dev/null 2>&1; then
-      dokku_log_warn "Container ${CID} still running"
-      continue
-    fi
-
-    DEAD_CONTAINERS+=("$CID")
-  done <"$DEAD_CONTAINER_FILE"
-
-  for CID in "${DEAD_CONTAINERS[@]}"; do
-    sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
-  done
+  fn-scheduler-docker-local-retire-containers
+  fn-scheduler-docker-local-retire-images
 }
 
 trigger-scheduler-docker-local-scheduler-retire "$@"