fix: make T3 route-removal assertions robust to reload race
T3 in every per-backend file polled the post-removal HTTP response using assert_http_localhost_response_contains, which retries on status code only. After a route is removed, both api and web return 200 for /api/v0/Procfile so the polling exits immediately; the body check then races with the proxy daemon picking up the new state and observes api's response (Procfile content) instead of web's catch-all. The proxy:route:remove output was also unechoed, hiding whether BuildConfig actually fired or was no-op suppressed. Each T3 now echoes the remove output and status, asserts via proxy:route:report that storage was cleared, then polls the body in a hand-rolled retry loop until the api marker is no longer present (timeout 30s). The k3s T3 files similarly retry the kubectl probe so the chart re-render has time to settle.
This commit is contained in:
6 files changed
+133
-19
No files matched your search
@@ -92,14 +92,32 @@ teardown() {
|
|||||||
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
||||||
|
|
||||||
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku proxy:route:report $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "(none)"
|
||||||
|
|
||||||
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
local body=""
|
||||||
run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
|
for attempt in $(seq 1 30); do
|
||||||
|
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
||||||
|
body="$output"
|
||||||
|
if ! grep -q 'python3 -m http.server' <<<"$body"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "final body attempt $attempt: $body"
|
||||||
|
run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
|
||||||
assert_output "0"
|
assert_output "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -109,14 +109,30 @@ teardown() {
|
|||||||
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
||||||
|
|
||||||
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
||||||
assert_success
|
|
||||||
|
|
||||||
# The rewrite-target Ingress should be gone after removal.
|
|
||||||
run /bin/bash -c "kubectl get ingress -o json | jq -r '[.items[] | select(.metadata.annotations[\"nginx.ingress.kubernetes.io/rewrite-target\"] != null)] | length'"
|
|
||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
assert_output "0"
|
|
||||||
|
# Confirm storage was cleared before checking cluster state.
|
||||||
|
run /bin/bash -c "dokku proxy:route:report $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "(none)"
|
||||||
|
|
||||||
|
# The rewrite-target Ingress should be gone after removal. Retry briefly
|
||||||
|
# in case the chart re-render hasn't settled.
|
||||||
|
local count="1"
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
run /bin/bash -c "kubectl get ingress -o json | jq -r '[.items[] | select(.metadata.annotations[\"nginx.ingress.kubernetes.io/rewrite-target\"] != null)] | length'"
|
||||||
|
count="$output"
|
||||||
|
if [[ "$count" == "0" ]]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "final ingress count after $attempt attempts: $count"
|
||||||
|
assert_equal "$count" "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
add_api_process_callback() {
|
add_api_process_callback() {
|
||||||
|
|||||||
@@ -109,13 +109,30 @@ teardown() {
|
|||||||
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
||||||
|
|
||||||
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
||||||
assert_success
|
|
||||||
|
|
||||||
run /bin/bash -c "kubectl get middleware.traefik.io -o json | jq -r '[.items[] | select(.spec.stripPrefix != null)] | length'"
|
|
||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
assert_output "0"
|
|
||||||
|
# Confirm storage was cleared before checking cluster state.
|
||||||
|
run /bin/bash -c "dokku proxy:route:report $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "(none)"
|
||||||
|
|
||||||
|
# The stripPrefix Middleware should be gone after removal. Retry briefly
|
||||||
|
# in case the chart re-render hasn't settled.
|
||||||
|
local count="1"
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
run /bin/bash -c "kubectl get middleware.traefik.io -o json | jq -r '[.items[] | select(.spec.stripPrefix != null)] | length'"
|
||||||
|
count="$output"
|
||||||
|
if [[ "$count" == "0" ]]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "final middleware count after $attempt attempts: $count"
|
||||||
|
assert_equal "$count" "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
add_api_process_callback() {
|
add_api_process_callback() {
|
||||||
|
|||||||
@@ -82,12 +82,35 @@ teardown() {
|
|||||||
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
||||||
|
|
||||||
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
# After removal, /api/v0/Procfile no longer reaches api - it falls through
|
# Confirm storage was actually mutated; a CommandRouteRemove that returns
|
||||||
# to web's catch-all. Assert the response is no longer api's Procfile body.
|
# changed=false silently skips BuildConfig and leaves nginx with the old
|
||||||
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
# route block.
|
||||||
run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
|
run /bin/bash -c "dokku proxy:route:report $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "(none)"
|
||||||
|
|
||||||
|
# nginx reload races with the curl after route removal: both api and web
|
||||||
|
# return 200 for /api/v0/Procfile so the polling loop in
|
||||||
|
# assert_http_localhost_response_contains exits on status alone before
|
||||||
|
# nginx picks up the new config. Hand-roll a retry that polls the BODY
|
||||||
|
# until it no longer contains api's marker.
|
||||||
|
local body=""
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
||||||
|
body="$output"
|
||||||
|
if ! grep -q 'python3 -m http.server' <<<"$body"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "final body attempt $attempt: $body"
|
||||||
|
run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
|
||||||
assert_output "0"
|
assert_output "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -92,14 +92,32 @@ teardown() {
|
|||||||
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
||||||
|
|
||||||
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku proxy:route:report $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "(none)"
|
||||||
|
|
||||||
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
local body=""
|
||||||
run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
|
for attempt in $(seq 1 30); do
|
||||||
|
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
||||||
|
body="$output"
|
||||||
|
if ! grep -q 'python3 -m http.server' <<<"$body"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "final body attempt $attempt: $body"
|
||||||
|
run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
|
||||||
assert_output "0"
|
assert_output "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -96,14 +96,36 @@ teardown() {
|
|||||||
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
assert_http_localhost_response_contains "http" "${TEST_APP}.${DOKKU_DOMAIN}" "80" "/api/v0/Procfile" "python3 -m http.server"
|
||||||
|
|
||||||
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
run /bin/bash -c "dokku proxy:route:remove $TEST_APP /api/v0"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
|
# Confirm storage was actually mutated before doing the slow rebuild.
|
||||||
|
run /bin/bash -c "dokku proxy:route:report $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "(none)"
|
||||||
|
|
||||||
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSso /tmp/route-removed-body "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
# After ps:rebuild, traefik recreates the api container without the route
|
||||||
run /bin/bash -c "grep -c 'python3 -m http.server' /tmp/route-removed-body || true"
|
# labels, but there is still a race with traefik picking up the new state.
|
||||||
|
# Hand-roll a retry that polls the BODY until api's marker is gone.
|
||||||
|
local body=""
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
run curl --connect-to "${TEST_APP}.${DOKKU_DOMAIN}:80:localhost:80" -kSs "http://${TEST_APP}.${DOKKU_DOMAIN}:80/api/v0/Procfile"
|
||||||
|
body="$output"
|
||||||
|
if ! grep -q 'python3 -m http.server' <<<"$body"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "final body attempt $attempt: $body"
|
||||||
|
run /bin/bash -c "grep -c 'python3 -m http.server' <<<\"$body\" || true"
|
||||||
assert_output "0"
|
assert_output "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user