Merge pull request #8507 from dokku/fix-8497-orphaned-containers-on-scale-down

Retire orphaned containers when scaling down

plugins/scheduler-docker-local/bin/scheduler-deploy-process

@@ -85,8 +85,15 @@ fn-scheduler-deploy-process() {
   else
     local CONTAINER_IDX_OFFSET=$((PROC_COUNT + 1))
   fi
-  local container_state_filetype
+  local container_state_filetype orphaned_container_file orphaned_cid
   pushd "$DOKKU_ROOT/$APP" >/dev/null
+  for orphaned_container_file in $(find . -maxdepth 1 -name "CONTAINER.$PROC_TYPE.*" -printf "%f\n" | sort -t . -k 3 -n | tail -n +$CONTAINER_IDX_OFFSET); do
+    [[ -s "$orphaned_container_file" ]] || continue
+    orphaned_cid="$(<"$orphaned_container_file")"
+    [[ -z "$orphaned_cid" ]] && continue
+    dokku_log_verbose "Scheduling orphaned container shutdown in $DOKKU_WAIT_TO_RETIRE seconds (${orphaned_container_file#CONTAINER.})"
+    plugn trigger scheduler-register-retired "$APP" "$orphaned_cid" "$DOKKU_WAIT_TO_RETIRE"
+  done
   for container_state_filetype in CONTAINER IP PORT; do
     find . -maxdepth 1 -name "$container_state_filetype.$PROC_TYPE.*" -printf "%f\n" | sort -t . -k 3 -n | tail -n +$CONTAINER_IDX_OFFSET | xargs rm -f
   done

tests/unit/scheduler-docker-local.bats

@@ -175,3 +175,54 @@ teardown() {
   assert_output_contains "Deploys may fail when publishing ports and scaling to multiple containers" 0
   assert_output_contains "Deploys may fail when publishing ports and enabling zero downtime" 0
 }
+
+@test "(scheduler-docker-local) scale down retires orphaned containers" {
+  run create_app
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku checks:set $TEST_APP wait-to-retire 1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:scale $TEST_APP web=2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  sleep 2
+
+  run /bin/bash -c "dokku ps:retire"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker ps --filter label=com.dokku.app-name=$TEST_APP --filter label=com.dokku.process-type=web --format '{{.Names}}' | wc -l"
+  echo "output: $output"
+  echo "status: $status"
+  assert_output "2"
+
+  run /bin/bash -c "dokku ps:scale $TEST_APP web=1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  sleep 2
+
+  run /bin/bash -c "dokku ps:retire"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker ps --filter label=com.dokku.app-name=$TEST_APP --filter label=com.dokku.process-type=web --format '{{.Names}}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_output "$TEST_APP.web.1"
+}