test: verify nginx returns 502 when coredns-docker is unavailable

Adds bats coverage on the bare-metal CI host (stops `coredns-docker` via `systemctl` and asserts the rendered vhost returns 502, then restarts and asserts recovery to 200) and a parallel `tests/test_coredns_outage` driver wired into `make test-ci-docker` that exercises the same flow against the dokku container using `sv stop|start coredns-docker`. Answers the open thread on whether the resolver-mode upstream surfaces a useful error when the DNS service is down.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-11-06 20:43:34 -05:00
1 parent 599bc7b243
commit ea72a80139
3 files changed
+136 -1

No files matched your search

+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env bash
set -xeo pipefail
SELF=$(command -v "$0")
TARGET="$1"
SCOPE="${2:-host}"
REMOTE="dokku@$TARGET"
APP="test-coredns-outage-$RANDOM"
cleanup() {
local exit_code=$?
set +e
start_coredns_docker
ssh "$REMOTE" apps:destroy "$APP" --force >/dev/null 2>&1
ssh "$REMOTE" events:off >/dev/null 2>&1
exit $exit_code
}
stop_coredns_docker() {
case "$SCOPE" in
docker)
docker exec dokku sv stop coredns-docker
;;
host)
sudo systemctl stop coredns-docker
;;
*)
echo "Unknown scope: $SCOPE"
exit 1
;;
esac
}
start_coredns_docker() {
case "$SCOPE" in
docker)
docker exec dokku sv start coredns-docker
;;
host)
sudo systemctl start coredns-docker
;;
*)
echo "Unknown scope: $SCOPE"
exit 1
;;
esac
}
curl_status() {
curl -kSso /dev/null -w "%{http_code}" -m 5 "http://$APP.$TARGET/"
}
wait_for_status() {
local expected="$1" attempts="${2:-15}" actual
while [[ $attempts -gt 0 ]]; do
actual="$(curl_status || true)"
[[ "$actual" == "$expected" ]] && return 0
attempts=$((attempts - 1))
sleep 1
done
echo "Expected HTTP $expected, last observed $actual"
return 1
}
trap cleanup INT TERM EXIT
ssh "$REMOTE" events:on
TMP=$(mktemp -d "/tmp/$TARGET.XXXXX")
rmdir "$TMP" && cp -r "$(dirname "$SELF")/apps/python" "$TMP"
pushd "$TMP" >/dev/null
git init -q
git config user.email "robot@example.com"
git config user.name "Test Robot"
git remote add target "${REMOTE}:${APP}"
[[ -f gitignore ]] && mv gitignore .gitignore
git add .
git commit -qm 'initial commit'
git push target master
popd >/dev/null
rm -rf "$TMP"
echo "-----> Verifying app responds before outage"
wait_for_status 200
echo "-----> Stopping coredns-docker ($SCOPE)"
stop_coredns_docker
# Wait long enough for any cached DNS entry (resolver valid=10s in the template)
sleep 12
echo "-----> Verifying nginx returns 502 with coredns-docker stopped"
actual="$(curl_status || true)"
if [[ "$actual" != "502" ]]; then
echo "Expected HTTP 502 with coredns-docker stopped, observed $actual"
exit 1
fi
echo "-----> Restarting coredns-docker"
start_coredns_docker
echo "-----> Verifying app recovers"
wait_for_status 200
echo "-----> coredns-docker outage test PASSED"
+25
View File
@@ -9,6 +9,7 @@ setup() {
}
teardown() {
sudo systemctl start coredns-docker >/dev/null 2>&1 || true
dokku --quiet nginx:set --global dns-resolver
dokku --quiet nginx:set --global dns-zone
destroy_app
@@ -137,3 +138,27 @@ teardown() {
assert_output_not_contains "\$dokku_upstream"
assert_output_contains "upstream ${TEST_APP}-"
}
@test "(nginx-vhosts) nginx returns 502 when coredns-docker is stopped" {
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
echo "output: $output"
echo "status: $status"
assert_success
run sudo systemctl stop coredns-docker
echo "output: $output"
echo "status: $status"
assert_success
run curl --connect-to "$TEST_APP.$DOKKU_DOMAIN:80:localhost:80" -kSso /dev/null -w "%{http_code}" -m 5 "http://$TEST_APP.$DOKKU_DOMAIN/"
echo "output: $output"
echo "status: $status"
assert_output "502"
run sudo systemctl start coredns-docker
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response http "$TEST_APP.$DOKKU_DOMAIN" 80 / "" 200
}