fix: make T3 route-removal assertions robust to reload race

T3 in every per-backend file polled the post-removal HTTP response using assert_http_localhost_response_contains, which retries on status code only. After a route is removed, both api and web return 200 for /api/v0/Procfile so the polling exits immediately; the body check then races with the proxy daemon picking up the new state and observes api's response (Procfile content) instead of web's catch-all. The proxy:route:remove output was also unechoed, hiding whether BuildConfig actually fired or was no-op suppressed. Each T3 now echoes the remove output and status, asserts via proxy:route:report that storage was cleared, then polls the body in a hand-rolled retry loop until the api marker is no longer present (timeout 30s). The k3s T3 files similarly retry the kubectl probe so the chart re-render has time to settle.

tests/unit/proxy-route-caddy.bats

@@ -92,14 +92,32 @@ teardown() {
   assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
 
   run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
+  echo "output: $output"
+  echo "status: $status"
   assert_success
+
+  run /bin/bash -c "dokku proxy:route:report $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "(none)"
+
   run /bin/bash -c "dokku ps:rebuild $TEST_APP"
   echo "output: $output"
   echo "status: $status"
   assert_success
 
-  run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
-  run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
+  local body=""
+  for attempt in $(seq 1 30); do
+    run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
+    body="$output"
+    if ! grep -q 'python3 -m http.server' <<<"$body"; then
+      break
+    fi
+    sleep 1
+  done
+  echo "final body attempt $attempt: $body"
+  run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
   assert_output "0"
 }
 

tests/unit/proxy-route-k3s-nginx.bats

@@ -109,14 +109,30 @@ teardown() {
   assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
 
   run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
-  assert_success
-
-  # The rewrite-target Ingress should be gone after removal.
-  run /bin/bash -c "kubectl get ingress -o json | jq -r '[.items[] | select(.metadata.annotations[\"nginx.ingress.kubernetes.io/rewrite-target\"] != null)] | length'"
   echo "output: $output"
   echo "status: $status"
   assert_success
-  assert_output "0"
+
+  # Confirm storage was cleared before checking cluster state.
+  run /bin/bash -c "dokku proxy:route:report $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "(none)"
+
+  # The rewrite-target Ingress should be gone after removal. Retry briefly
+  # in case the chart re-render hasn't settled.
+  local count="1"
+  for attempt in $(seq 1 30); do
+    run /bin/bash -c "kubectl get ingress -o json | jq -r '[.items[] | select(.metadata.annotations[\"nginx.ingress.kubernetes.io/rewrite-target\"] != null)] | length'"
+    count="$output"
+    if [[ "$count" == "0" ]]; then
+      break
+    fi
+    sleep 1
+  done
+  echo "final ingress count after $attempt attempts: $count"
+  assert_equal "$count" "0"
 }
 
 add_api_process_callback() {

tests/unit/proxy-route-k3s-traefik.bats

@@ -109,13 +109,30 @@ teardown() {
   assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
 
   run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
-  assert_success
-
-  run /bin/bash -c "kubectl get middleware.traefik.io -o json | jq -r '[.items[] | select(.spec.stripPrefix != null)] | length'"
   echo "output: $output"
   echo "status: $status"
   assert_success
-  assert_output "0"
+
+  # Confirm storage was cleared before checking cluster state.
+  run /bin/bash -c "dokku proxy:route:report $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "(none)"
+
+  # The stripPrefix Middleware should be gone after removal. Retry briefly
+  # in case the chart re-render hasn't settled.
+  local count="1"
+  for attempt in $(seq 1 30); do
+    run /bin/bash -c "kubectl get middleware.traefik.io -o json | jq -r '[.items[] | select(.spec.stripPrefix != null)] | length'"
+    count="$output"
+    if [[ "$count" == "0" ]]; then
+      break
+    fi
+    sleep 1
+  done
+  echo "final middleware count after $attempt attempts: $count"
+  assert_equal "$count" "0"
 }
 
 add_api_process_callback() {

tests/unit/proxy-route-nginx.bats

@@ -82,12 +82,35 @@ teardown() {
   assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
 
   run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
+  echo "output: $output"
+  echo "status: $status"
   assert_success
 
-  # After removal, /api/v0/Procfile no longer reaches api - it falls through
-  # to web's catch-all. Assert the response is no longer api's Procfile body.
-  run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
-  run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
+  # Confirm storage was actually mutated; a CommandRouteRemove that returns
+  # changed=false silently skips BuildConfig and leaves nginx with the old
+  # route block.
+  run /bin/bash -c "dokku proxy:route:report $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "(none)"
+
+  # nginx reload races with the curl after route removal: both api and web
+  # return 200 for /api/v0/Procfile so the polling loop in
+  # assert_http_localhost_response_contains exits on status alone before
+  # nginx picks up the new config. Hand-roll a retry that polls the BODY
+  # until it no longer contains api's marker.
+  local body=""
+  for attempt in $(seq 1 30); do
+    run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
+    body="$output"
+    if ! grep -q 'python3 -m http.server' <<<"$body"; then
+      break
+    fi
+    sleep 1
+  done
+  echo "final body attempt $attempt: $body"
+  run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
   assert_output "0"
 }
 

tests/unit/proxy-route-openresty.bats

@@ -92,14 +92,32 @@ teardown() {
   assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
 
   run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
+  echo "output: $output"
+  echo "status: $status"
   assert_success
+
+  run /bin/bash -c "dokku proxy:route:report $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "(none)"
+
   run /bin/bash -c "dokku ps:rebuild $TEST_APP"
   echo "output: $output"
   echo "status: $status"
   assert_success
 
-  run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
-  run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
+  local body=""
+  for attempt in $(seq 1 30); do
+    run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
+    body="$output"
+    if ! grep -q 'python3 -m http.server' <<<"$body"; then
+      break
+    fi
+    sleep 1
+  done
+  echo "final body attempt $attempt: $body"
+  run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
   assert_output "0"
 }
 

tests/unit/proxy-route-traefik.bats

@@ -96,14 +96,36 @@ teardown() {
   assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
 
   run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
+  echo "output: $output"
+  echo "status: $status"
   assert_success
+
+  # Confirm storage was actually mutated before doing the slow rebuild.
+  run /bin/bash -c "dokku proxy:route:report $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "(none)"
+
   run /bin/bash -c "dokku ps:rebuild $TEST_APP"
   echo "output: $output"
   echo "status: $status"
   assert_success
 
-  run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
-  run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
+  # After ps:rebuild, traefik recreates the api container without the route
+  # labels, but there is still a race with traefik picking up the new state.
+  # Hand-roll a retry that polls the BODY until api's marker is gone.
+  local body=""
+  for attempt in $(seq 1 30); do
+    run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
+    body="$output"
+    if ! grep -q 'python3 -m http.server' <<<"$body"; then
+      break
+    fi
+    sleep 1
+  done
+  echo "final body attempt $attempt: $body"
+  run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
   assert_output "0"
 }