Merge pull request #9180 from dokku/9168-scheduler-k3s-deployment-retry-fails-cleaning-up-a-non-empty-kustomization-directory

Clean up stale extracted files on deploy

plugins/common/common.go

@@ -153,8 +153,8 @@ func CorePostDeploy(input CorePostDeployInput) error {
 				if err := os.Rename(processSpecificPath, existingPath); err != nil {
 					return err
 				}
-			} else if DirectoryExists(fmt.Sprintf("%s.missing", processSpecificPath)) {
+			} else if FileExists(fmt.Sprintf("%s.missing", processSpecificPath)) {
-				if err := os.RemoveAll(fmt.Sprintf("%s.missing", processSpecificPath)); err != nil {
+				if err := os.Remove(fmt.Sprintf("%s.missing", processSpecificPath)); err != nil {
 					return err
 				}
 
@@ -306,7 +306,7 @@ func CorePostExtract(input CorePostExtractInput) error {
 			return err
 		}
 		for _, f := range files {
-			if err := os.Remove(f); err != nil {
+			if err := os.RemoveAll(f); err != nil {
 				return err
 			}
 		}
@@ -371,13 +371,18 @@ func CorePostExtract(input CorePostExtractInput) error {
 			}
 		} else {
 			if toExtract.IsDirectory {
-
 				if err := CopyDirFromImage(input.AppName, sourceImage, imageSourcePath, processSpecificDestination); err != nil {
-					return TouchFile(missingDestination)
+					if err := TouchFile(missingDestination); err != nil {
+						return err
+					}
+					continue
 				}
 			} else {
 				if err := CopyFromImage(input.AppName, sourceImage, imageSourcePath, processSpecificDestination); err != nil {
-					return TouchFile(missingDestination)
+					if err := TouchFile(missingDestination); err != nil {
+						return err
+					}
+					continue
 				}
 			}
 		}

plugins/nginx-vhosts/core-post-extract

@@ -8,8 +8,6 @@ set -eo pipefail
 fn-nginx-vhosts-copy-from-image() {
   declare APP="$1" IMAGE_NAME="$2" CONF_SIGIL_PATH="$3"
 
-  mkdir -p "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP"
-  find "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/" -maxdepth 1 -type f -name 'nginx.conf.sigil.*' -delete
   copy_from_image "$IMAGE_NAME" "$CONF_SIGIL_PATH" "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID" 2>/dev/null || true
   if [[ ! -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID" ]]; then
     touch "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID.missing"
@@ -20,7 +18,6 @@ fn-nginx-vhosts-copy-from-directory() {
   declare APP="$1" SOURCECODE_WORK_DIR="$2" CONF_SIGIL_PATH="$3"
 
   pushd "$SOURCECODE_WORK_DIR" >/dev/null
-  mkdir -p "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP"
 
   if [[ -z "$CONF_SIGIL_PATH" ]]; then
     touch "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID.missing"
@@ -32,7 +29,6 @@ fn-nginx-vhosts-copy-from-directory() {
     return
   fi
 
-  find "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/" -maxdepth 1 -type f -name 'nginx.conf.sigil.*' -delete
   cp -f "$CONF_SIGIL_PATH" "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID"
   popd &>/dev/null || pushd "/tmp" >/dev/null
 }
@@ -44,6 +40,9 @@ trigger-nginx-vhosts-core-post-extract() {
   local CONF_SIGIL_PATH="$(fn-nginx-computed-nginx-conf-sigil-path "$APP")"
   local app_source_image
 
+  mkdir -p "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP"
+  find "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/" -maxdepth 1 -type f -name 'nginx.conf.sigil.*' -delete
+
   app_source_image="$(plugn trigger git-get-property "$APP" "source-image")"
   if [[ -n "$app_source_image" ]]; then
     fn-nginx-vhosts-copy-from-image "$APP" "$app_source_image" "$CONF_SIGIL_PATH"

plugins/openresty-vhosts/core-post-deploy

@@ -4,25 +4,34 @@ source "$PLUGIN_AVAILABLE_PATH/openresty-vhosts/internal-functions"
 set -eo pipefail
 [[ $DOKKU_TRACE ]] && set -x
 
+fn-openresty-vhosts-promote-includes-dir() {
+  declare desc="moves the process-specific includes dir into place or removes the stored one"
+  declare APP="$1" INCLUDES_NAME="$2"
+  local includes_dir="${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/$INCLUDES_NAME"
+
+  if [[ -d "$includes_dir.$DOKKU_PID" ]]; then
+    rm -rf "$includes_dir"
+    mv "$includes_dir.$DOKKU_PID" "$includes_dir"
+  elif [[ -f "$includes_dir.$DOKKU_PID.missing" ]]; then
+    rm -f "$includes_dir.$DOKKU_PID.missing"
+    rm -rf "$includes_dir"
+  fi
+}
+
 trigger-openresty-vhosts-core-post-deploy() {
   declare desc="openresty-vhosts core-post-deploy plugin trigger"
   declare trigger="core-post-deploy"
   declare APP="$1"
   local tls_internal
 
+  fn-openresty-vhosts-promote-includes-dir "$APP" "openresty-http-includes"
+  fn-openresty-vhosts-promote-includes-dir "$APP" "openresty-location-includes"
+
   if [[ "$(plugn trigger proxy-type "$APP")" != "openresty" ]]; then
     return
   fi
 
   dokku_log_info1 "Routing app via openresty"
-  if [[ -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID.missing" ]]; then
-    rm -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID.missing"
-    rm -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes"
-  fi
-  if [[ -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID.missing" ]]; then
-    rm -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID.missing"
-    rm -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes"
-  fi
 }
 
 trigger-openresty-vhosts-core-post-deploy "$@"

plugins/openresty-vhosts/core-post-extract

@@ -45,18 +45,15 @@ fn-openresty-vhosts-copy-from-image() {
   local CONF_PATH="openresty/http-includes"
   local LOCATION_CONF_PATH="openresty/http-location-includes"
 
-  mkdir -p "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP"
-  find "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/" -maxdepth 1 -name 'openresty-http-includes.*' -type d -exec rm -r {} +
   copy_dir_from_image "$IMAGE_NAME" "$CONF_PATH" "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID" 2>/dev/null || true
   fn-openresty-vhosts-sanitize-extracted-includes "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID"
-  if [[ ! -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID" ]]; then
+  if [[ ! -d "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID" ]]; then
     touch "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID.missing"
   fi
 
-  find "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/" -maxdepth 1 -name 'openresty-location-includes.*' -type d -exec rm -r {} +
   copy_dir_from_image "$IMAGE_NAME" "$LOCATION_CONF_PATH" "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID" 2>/dev/null || true
   fn-openresty-vhosts-sanitize-extracted-includes "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID"
-  if [[ ! -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID" ]]; then
+  if [[ ! -d "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID" ]]; then
     touch "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID.missing"
   fi
 }
@@ -66,25 +63,21 @@ fn-openresty-vhosts-copy-from-directory() {
   local CONF_PATH="openresty/http-includes"
   local LOCATION_CONF_PATH="openresty/http-location-includes"
 
+  pushd "$SOURCECODE_WORK_DIR" >/dev/null
   if [[ -d "$CONF_PATH" ]]; then
-    pushd "$SOURCECODE_WORK_DIR" >/dev/null
-    find "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/" -maxdepth 1 -name 'openresty-http-includes.*' -type d -exec rm -r {} +
     mkdir -p "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID/"
     fn-openresty-vhosts-copy-includes-from-source "$CONF_PATH" "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID"
-    popd &>/dev/null || pushd "/tmp" >/dev/null
   else
     touch "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID.missing"
   fi
 
   if [[ -d "$LOCATION_CONF_PATH" ]]; then
-    pushd "$SOURCECODE_WORK_DIR" >/dev/null
-    find "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/" -maxdepth 1 -name 'openresty-location-includes.*' -type d -exec rm -r {} +
     mkdir -p "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID/"
     fn-openresty-vhosts-copy-includes-from-source "$LOCATION_CONF_PATH" "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID"
-    popd &>/dev/null || pushd "/tmp" >/dev/null
   else
     touch "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID.missing"
   fi
+  popd &>/dev/null || pushd "/tmp" >/dev/null
 }
 
 trigger-openresty-vhosts-core-post-extract() {
@@ -94,6 +87,7 @@ trigger-openresty-vhosts-core-post-extract() {
   local app_source_image
 
   mkdir -p "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP"
+  find "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/" -maxdepth 1 \( -name 'openresty-http-includes.*' -o -name 'openresty-location-includes.*' \) -exec rm -rf {} +
 
   app_source_image="$(plugn trigger git-get-property "$APP" "source-image")"
   if [[ -n "$app_source_image" ]]; then

plugins/openresty-vhosts/internal-functions

@@ -221,7 +221,7 @@ fn-openresty-get-http-includes-dir() {
   declare desc="get any include dir if available"
   declare APP="$1"
 
-  if [[ -d "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID.missing" ]]; then
+  if [[ -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-http-includes.$DOKKU_PID.missing" ]]; then
     return
   fi
 
@@ -236,7 +236,7 @@ fn-openresty-get-location-includes-dir() {
   declare desc="get any include dir if available"
   declare APP="$1"
 
-  if [[ -d "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID.missing" ]]; then
+  if [[ -f "${DOKKU_LIB_ROOT}/data/openresty-vhosts/app-$APP/openresty-location-includes.$DOKKU_PID.missing" ]]; then
     return
   fi
 

plugins/scheduler-docker-local/core-post-deploy

@@ -10,11 +10,6 @@ trigger-scheduler-docker-local-core-post-deploy() {
   declare APP="$1"
   local APP_ROOT="$DOKKU_ROOT/$APP"
 
-  local DOKKU_SCHEDULER=$(get_app_scheduler "$APP")
-  if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then
-    return
-  fi
-
   # move over the checks file appropriately
   checks_file="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS"
   process_specific_checks_file="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS.$DOKKU_PID"
@@ -27,6 +22,11 @@ trigger-scheduler-docker-local-core-post-deploy() {
     fi
   fi
 
+  local DOKKU_SCHEDULER=$(get_app_scheduler "$APP")
+  if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then
+    return
+  fi
+
   dokku_log_info1 "Renaming containers"
   local PROCTYPES="$(plugn trigger ps-current-scale "$APP" | awk -F '=' '{ print $1 }' | xargs)"
   local CONTAINER_FILES="$(find "$DOKKU_ROOT/$APP" -maxdepth 1 -name "CONTAINER.*" -printf "%f\n" 2>/dev/null | sort -t . -k 3 -n | xargs)"

plugins/scheduler-docker-local/core-post-extract

@@ -8,8 +8,6 @@ source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
 fn-scheduler-docker-local-copy-from-image() {
   declare APP="$1" IMAGE_NAME="$2" CHECKS_PATH="$3"
 
-  mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP"
-  find "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/" -maxdepth 1 -type f -name 'CHECKS.*' -delete
   copy_from_image "$IMAGE_NAME" "$CHECKS_PATH" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS.$DOKKU_PID" 2>/dev/null || true
   if [[ ! -f "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS.$DOKKU_PID" ]]; then
     touch "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS.$DOKKU_PID.missing"
@@ -20,7 +18,6 @@ fn-scheduler-docker-local-copy-from-directory() {
   declare APP="$1" SOURCECODE_WORK_DIR="$2" CHECKS_PATH="$3"
 
   pushd "$SOURCECODE_WORK_DIR" >/dev/null
-  mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP"
 
   if [[ -z "$CHECKS_PATH" ]]; then
     touch "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS.$DOKKU_PID.missing"
@@ -32,7 +29,6 @@ fn-scheduler-docker-local-copy-from-directory() {
     return
   fi
 
-  find "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/" -maxdepth 1 -type f -name 'CHECKS.*' -delete
   cp -f "$CHECKS_PATH" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS.$DOKKU_PID"
   popd &>/dev/null || pushd "/tmp" >/dev/null
 }
@@ -44,6 +40,9 @@ trigger-scheduler-docker-local-core-post-extract() {
   local CHECKS_PATH="CHECKS" CHECKS_FILENAME="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/CHECKS.$DOKKU_PID"
   local app_source_image
 
+  mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP"
+  find "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/" -maxdepth 1 -type f -name 'CHECKS.*' -delete
+
   app_source_image="$(plugn trigger git-get-property "$APP" "source-image")"
   if [[ -n "$app_source_image" ]]; then
     fn-scheduler-docker-local-copy-from-image "$APP" "$app_source_image" "$CHECKS_PATH"

plugins/scheduler-k3s/functions.go

@@ -2144,7 +2144,7 @@ func getProcessSpecificKustomizeRootPath(appName string) string {
 func hasKustomizeDirectory(appName string) bool {
 	directory := getKustomizeDirectory(appName)
 
-	if common.DirectoryExists(fmt.Sprintf("%s.%s.missing", directory, os.Getenv("DOKKU_PID"))) {
+	if common.FileExists(fmt.Sprintf("%s.%s.missing", directory, os.Getenv("DOKKU_PID"))) {
 		return false
 	}
 

tests/unit/nginx-vhosts_12.bats

@@ -129,3 +129,22 @@ teardown() {
 
   docker image rm dokku-test/$TEST_APP:latest dokku-test/$TEST_APP:v2 || true
 }
+
+@test "(nginx-vhosts) deploy removes stale nginx.conf.sigil files" {
+  local DATA_DIR="/var/lib/dokku/data/nginx-vhosts/app-$TEST_APP"
+  mkdir -p "$DATA_DIR"
+  echo "# stale" >"$DATA_DIR/nginx.conf.sigil.99999999"
+  touch "$DATA_DIR/nginx.conf.sigil.99999998.missing"
+  chown -R dokku:dokku "$DATA_DIR"
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "find $DATA_DIR -maxdepth 1 -name 'nginx.conf.sigil.*'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_exists
+}

tests/unit/openresty-includes.bats

@@ -0,0 +1,136 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  global_setup
+  dokku nginx:stop
+  dokku openresty:set --global letsencrypt-server https://acme-staging-v02.api.letsencrypt.org/directory
+  dokku openresty:set --global letsencrypt-email
+  dokku openresty:start
+  create_app
+}
+
+teardown() {
+  global_teardown
+  destroy_app
+  dokku openresty:stop
+  dokku nginx:start
+}
+
+@test "(openresty) includes persist across ps:restart" {
+  run /bin/bash -c "dokku proxy:set $TEST_APP openresty"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_openresty_include
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.include-http-example.conf\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "IyBmb3JjZSB0aGUgY2hhcmFjdGVyIHNldCB0byB1dGYtOApjaGFyc2V0IFVURi04Owo="
+
+  run /bin/bash -c "dokku ps:restart $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.include-http-example.conf\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "IyBmb3JjZSB0aGUgY2hhcmFjdGVyIHNldCB0byB1dGYtOApjaGFyc2V0IFVURi04Owo="
+
+  run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.include-location-http-example.conf\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "IyBsb2NhdGlvbi1ibG9jawo="
+}
+
+@test "(openresty) includes are removed when deleted from the repo" {
+  local DATA_DIR="/var/lib/dokku/data/openresty-vhosts/app-$TEST_APP"
+  local CUSTOM_TMP=$(mktemp -d "/tmp/${DOKKU_DOMAIN}.XXXXX")
+  trap 'popd &>/dev/null || true; rm -rf "$CUSTOM_TMP"' INT TERM
+
+  run /bin/bash -c "dokku proxy:set $TEST_APP openresty"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  CUSTOM_TMP="$CUSTOM_TMP" run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_openresty_include
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run git -C "$CUSTOM_TMP" rm -r openresty
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run git -C "$CUSTOM_TMP" commit -m 'Remove openresty includes'
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run git -C "$CUSTOM_TMP" push target master:master
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.include-http-example.conf\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_exists
+
+  run /bin/bash -c "find $DATA_DIR -maxdepth 1 -name 'openresty-*-includes*'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_exists
+
+  rm -rf "$CUSTOM_TMP"
+}
+
+@test "(openresty) deploy removes stale include markers and directories" {
+  local DATA_DIR="/var/lib/dokku/data/openresty-vhosts/app-$TEST_APP"
+  mkdir -p "$DATA_DIR/openresty-http-includes.99999999" "$DATA_DIR/openresty-location-includes.99999999"
+  echo "charset UTF-8;" >"$DATA_DIR/openresty-http-includes.99999999/example.conf"
+  echo "# location-block" >"$DATA_DIR/openresty-location-includes.99999999/example.conf"
+  touch "$DATA_DIR/openresty-http-includes.99999998.missing" "$DATA_DIR/openresty-location-includes.99999998.missing"
+  chown -R dokku:dokku "$DATA_DIR"
+
+  run /bin/bash -c "dokku proxy:set $TEST_APP openresty"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "find $DATA_DIR -maxdepth 1 -name 'openresty-*-includes*'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_exists
+}
+
+add_openresty_include() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  mkdir -p "$APP_REPO_DIR/openresty/http-includes"
+  echo "# force the character set to utf-8" >>"$APP_REPO_DIR/openresty/http-includes/example.conf"
+  echo "charset UTF-8;" >>"$APP_REPO_DIR/openresty/http-includes/example.conf"
+
+  mkdir -p "$APP_REPO_DIR/openresty/http-location-includes"
+  echo "# location-block" >>"$APP_REPO_DIR/openresty/http-location-includes/example.conf"
+}

tests/unit/scheduler-docker-local.bats

@@ -588,6 +588,31 @@ teardown() {
   assert_output_contains "Unable to parse command"
 }
 
+@test "(scheduler-docker-local) deploy removes stale CHECKS files" {
+  local DATA_DIR="/var/lib/dokku/data/scheduler-docker-local/$TEST_APP"
+
+  run create_app
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  mkdir -p "$DATA_DIR"
+  echo "/ python/http.server" >"$DATA_DIR/CHECKS.99999999"
+  touch "$DATA_DIR/CHECKS.99999998.missing"
+  chown -R dokku:dokku "$DATA_DIR"
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "find $DATA_DIR -maxdepth 1 -name 'CHECKS.*'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_exists
+}
+
 quoted_procfile_callback() {
   local APP="$1"
   local APP_REPO_DIR="$2"

tests/unit/scheduler-k3s-kustomize.bats

@@ -0,0 +1,88 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  global_setup
+  create_app
+}
+
+teardown() {
+  destroy_app
+  global_teardown
+}
+
+@test "(scheduler-k3s) [kustomize] deploy removes a stale non-empty kustomization directory" {
+  local DATA_DIR="/var/lib/dokku/data/scheduler-k3s/$TEST_APP"
+  mkdir -p "$DATA_DIR/kustomization.99999999"
+  echo "kind: Kustomization" >"$DATA_DIR/kustomization.99999999/kustomization.yaml"
+  chown -R dokku:dokku "$DATA_DIR"
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "find $DATA_DIR -maxdepth 1 -name 'kustomization.*'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_exists
+}
+
+@test "(scheduler-k3s) [kustomize] deploy removes the stored kustomization when the repo no longer has one" {
+  local DATA_DIR="/var/lib/dokku/data/scheduler-k3s/$TEST_APP"
+  local CUSTOM_TMP=$(mktemp -d "/tmp/${DOKKU_DOMAIN}.XXXXX")
+  trap 'popd &>/dev/null || true; rm -rf "$CUSTOM_TMP"' INT TERM
+
+  CUSTOM_TMP="$CUSTOM_TMP" run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP inject_kustomization_yaml
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "test -f $DATA_DIR/kustomization/kustomization.yaml"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run git -C "$CUSTOM_TMP" rm -r config/kustomize
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run git -C "$CUSTOM_TMP" commit -m 'Remove kustomization'
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run git -C "$CUSTOM_TMP" push target master:master
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "test -e $DATA_DIR/kustomization"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "find $DATA_DIR -maxdepth 1 -name 'kustomization.*'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_exists
+
+  rm -rf "$CUSTOM_TMP"
+}
+
+inject_kustomization_yaml() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  mkdir -p "$APP_REPO_DIR/config/kustomize"
+  cat <<EOF >"$APP_REPO_DIR/config/kustomize/kustomization.yaml"
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- rendered.yaml
+EOF
+}