From 0786954d34aca76c49c7d8550ad9788f79fc97fd Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 26 Nov 2020 03:21:45 -0500
Subject: [PATCH] feat: schedule related images for cleanup when retiring
 containers

If the containers no longer exist and the images are unused, we should remove the images as well.

Note that images may sometimes stick around for a little while, depending on the retire schedule and the wait time specified for the container.

Closes #4104
---
 docs/development/plugin-triggers.md           |   2 +-
 .../scheduler-docker-local/internal-functions | 110 +++++++++++++++++-
 .../scheduler-docker-local/scheduler-deploy   |   2 +-
 .../scheduler-register-retired                |   6 +-
 .../scheduler-docker-local/scheduler-retire   |  44 +------
 5 files changed, 116 insertions(+), 48 deletions(-)

diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md
index 53ea488ec..fab7d8406 100644
--- a/docs/development/plugin-triggers.md
+++ b/docs/development/plugin-triggers.md
@@ -1914,7 +1914,7 @@ DOKKU_SCHEDULER="$1"; APP="$2"; CONTAINER_ID="$3";
 > Warning: The scheduler plugin trigger apis are under development and may change
 > between minor releases until the 1.0 release.
 
-- Description: Allows scheduling retiring a local container
+- Description: Allows scheduling retiring a local container and any related images
 - Invoked by: `internally`
 - Arguments: `$APP $CONTAINER_ID`
 - Example:
diff --git a/plugins/scheduler-docker-local/internal-functions b/plugins/scheduler-docker-local/internal-functions
index 030714df3..9f01be4e7 100755
--- a/plugins/scheduler-docker-local/internal-functions
+++ b/plugins/scheduler-docker-local/internal-functions
@@ -105,12 +105,116 @@ fn-scheduler-docker-local-retire-container() {
   fi
 }
 
-fn-scheduler-docker-local-register-retired-container() {
-  declare APP="$1" CID="$2" WAIT="$3"
+fn-scheduler-docker-local-retire-containers() {
   local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
+  local APP CID CURRENT_TIME DEAD_TIME STATE
+
+  if [[ ! -f "$DEAD_CONTAINER_FILE" ]]; then
+    return
+  fi
+
+  DEAD_CONTAINERS=()
+  while read line; do
+    [[ -z "$line" ]] && continue
+    CURRENT_TIME="$(date +%s)"
+    APP="$(echo "$line" | cut -d ' ' -f1)"
+    CID="$(echo "$line" | cut -d ' ' -f2)"
+    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
+
+    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
+      continue
+    fi
+
+    fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
+    STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
+    if [[ -z "$STATE" ]]; then
+      DEAD_CONTAINERS+=("$CID")
+      continue
+    fi
+
+    if [[ "$STATE" == "running" ]]; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    "$DOCKER_BIN" container rm --force "$CID" >/dev/null 2>&1 || true
+    if "$DOCKER_BIN" container inspect "${CID}" >/dev/null 2>&1; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    DEAD_CONTAINERS+=("$CID")
+  done <"$DEAD_CONTAINER_FILE"
+
+  for CID in "${DEAD_CONTAINERS[@]}"; do
+    sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
+  done
+}
+
+fn-scheduler-docker-local-retire-images() {
+  local DEAD_IMAGE_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
+  local APP IMAGE_ID CURRENT_TIME DEAD_TIME STATE RM_OUTPUT
+
+  if [[ ! -f "$DEAD_IMAGE_FILE" ]]; then
+    return
+  fi
+
+  DEAD_IMAGES=()
+  while read line; do
+    [[ -z "$line" ]] && continue
+    CURRENT_TIME="$(date +%s)"
+    APP="$(echo "$line" | cut -d ' ' -f1)"
+    IMAGE_ID="$(echo "$line" | cut -d ' ' -f2)"
+    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
+
+    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
+      continue
+    fi
+
+    STATE="$("$DOCKER_BIN" image inspect --format "{{ .Id }}" "$IMAGE_ID" 2>/dev/null || true)"
+    if [[ -z "$STATE" ]]; then
+      DEAD_IMAGES+=("$IMAGE_ID")
+      continue
+    fi
+
+    dokku_log_info1_quiet "Attempting to retire $APP image $IMAGE_ID"
+    if RM_OUTPUT="$("$DOCKER_BIN" image rm "$IMAGE_ID" 2>&1)"; then
+      DEAD_IMAGES+=("$IMAGE_ID")
+      continue
+    fi
+
+    if echo "$RM_OUTPUT" | grep -q "image has dependent child images"; then
+      dokku_log_warn "Image ${IMAGE_ID} has children images, skipping rm and marking dead"
+      DEAD_IMAGES+=("$IMAGE_ID")
+      continue
+    fi
+
+    if echo "$RM_OUTPUT" | grep -q "image is being used by running container"; then
+      dokku_log_warn "Image ${IMAGE_ID} has running containers, skipping rm"
+      continue
+    fi
+
+    dokku_log_warn "Image ${IMAGE_ID} still running"
+  done <"$DEAD_IMAGE_FILE"
+
+  for IMAGE_ID in "${DEAD_IMAGES[@]}"; do
+    sed -i "/${IMAGE_ID}/d" "$DEAD_IMAGE_FILE"
+  done
+
+  sort -o "$DEAD_IMAGE_FILE" -r "$DEAD_IMAGE_FILE"
+}
+
+fn-scheduler-docker-local-register-retired() {
+  declare TYPE="$1" APP="$2" DOCKER_ID="$3" WAIT="$4"
+  local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
+  if [[ "$TYPE" == "image" ]]; then
+    local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
+  fi
   local CURRENT_TIME DEAD_TIME
 
   CURRENT_TIME="$(date +%s)"
   DEAD_TIME=$((CURRENT_TIME + WAIT))
-  echo "${APP} ${CID} ${DEAD_TIME}" >>"${DEAD_CONTAINER_FILE}"
+  if ! grep -q "$DOCKER_ID" "$DEAD_FILE"; then
+    echo "${APP} ${DOCKER_ID} ${DEAD_TIME}" >>"${DEAD_FILE}"
+  fi
 }
diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy
index cabfc8146..9613ae12d 100755
--- a/plugins/scheduler-docker-local/scheduler-deploy
+++ b/plugins/scheduler-docker-local/scheduler-deploy
@@ -180,7 +180,7 @@ trigger-scheduler-docker-local-scheduler-deploy() {
     local oldid
     for oldid in $oldids; do
       dokku_log_verbose_quiet "$oldid"
-      fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT"
+      plugn trigger scheduler-register-retired "$APP" "$oldid" "$WAIT"
     done
     (
       exec >/dev/null 2>/dev/null </dev/null
diff --git a/plugins/scheduler-docker-local/scheduler-register-retired b/plugins/scheduler-docker-local/scheduler-register-retired
index 4b1882668..f0e520d26 100755
--- a/plugins/scheduler-docker-local/scheduler-register-retired
+++ b/plugins/scheduler-docker-local/scheduler-register-retired
@@ -7,8 +7,12 @@ trigger-scheduler-docker-local-scheduler-register-retired() {
   declare desc="register a container for retiring"
   declare trigger="scheduler-register-retired"
   declare APP="$1" CONTAINER_ID="$2" WAIT="${3:-60}"
+  local IMAGE_ID
+
+  IMAGE_ID="$("$DOCKER_BIN" container inspect "$CONTAINER_ID" --format "{{.Image}}" | cut -d: -f2)"
+  fn-scheduler-docker-local-register-retired "image" "$APP" "$IMAGE_ID" "$WAIT"
+  fn-scheduler-docker-local-register-retired "container" "$APP" "$CONTAINER_ID" "$WAIT"
 
-  fn-scheduler-docker-local-register-retired-container "$APP" "$CONTAINER_ID" "$WAIT"
 }
 
 trigger-scheduler-docker-local-scheduler-register-retired "$@"
diff --git a/plugins/scheduler-docker-local/scheduler-retire b/plugins/scheduler-docker-local/scheduler-retire
index d81f24fa0..cf469956b 100755
--- a/plugins/scheduler-docker-local/scheduler-retire
+++ b/plugins/scheduler-docker-local/scheduler-retire
@@ -8,48 +8,8 @@ trigger-scheduler-docker-local-scheduler-retire() {
   declare desc="retires all old containers once they have aged out"
   declare trigger="scheduler-retire"
 
-  local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
-  local APP CID CURRENT_TIME DEAD_TIME STATE
-
-  if [[ ! -f "$DEAD_CONTAINER_FILE" ]]; then
-    return
-  fi
-
-  DEAD_CONTAINERS=()
-  while read line; do
-    CURRENT_TIME="$(date +%s)"
-    APP="$(echo "$line" | cut -d ' ' -f1)"
-    CID="$(echo "$line" | cut -d ' ' -f2)"
-    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
-
-    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
-      continue
-    fi
-
-    fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
-    STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
-    if [[ -z "$STATE" ]]; then
-      DEAD_CONTAINERS+=("$CID")
-      continue
-    fi
-
-    if [[ "$STATE" == "running" ]]; then
-      dokku_log_warn "Container ${CID} still running"
-      continue
-    fi
-
-    "$DOCKER_BIN" container rm --force "$CID" >/dev/null 2>&1 || true
-    if "$DOCKER_BIN" container inspect "${CID}" >/dev/null 2>&1; then
-      dokku_log_warn "Container ${CID} still running"
-      continue
-    fi
-
-    DEAD_CONTAINERS+=("$CID")
-  done <"$DEAD_CONTAINER_FILE"
-
-  for CID in "${DEAD_CONTAINERS[@]}"; do
-    sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
-  done
+  fn-scheduler-docker-local-retire-containers
+  fn-scheduler-docker-local-retire-images
 }
 
 trigger-scheduler-docker-local-scheduler-retire "$@"