diff --git a/tests.mk b/tests.mk index da1230b1a..b0e96dd57 100644 --- a/tests.mk +++ b/tests.mk @@ -239,6 +239,10 @@ deploy-test-config: @echo deploying config app... cd tests && ./test_deploy ./apps/config $(DOKKU_DOMAIN) +deploy-test-coredns-docker-outage: + @echo testing nginx-vhosts behavior when coredns-docker is unavailable... + cd tests && ./test_coredns_outage $(DOKKU_DOMAIN) docker + deploy-test-dockerfile: @echo deploying dockerfile app... cd tests && ./test_deploy ./apps/dockerfile $(DOKKU_DOMAIN) @@ -342,7 +346,7 @@ deploy-test-storage-lib-host-root: @echo deploy-test-storage-lib-host-root cd tests/docker && bats storage.bats -test-ci-docker: setup-docker-deploy-tests deploy-test-checks-root deploy-test-config deploy-test-multi deploy-test-go-fail-predeploy deploy-test-go-fail-postdeploy deploy-test-storage-lib-host-root test-ci-binaries +test-ci-docker: setup-docker-deploy-tests deploy-test-checks-root deploy-test-config deploy-test-multi deploy-test-go-fail-predeploy deploy-test-go-fail-postdeploy deploy-test-storage-lib-host-root deploy-test-coredns-docker-outage test-ci-binaries generate-ssl-tars: generate-ssl-tar generate-ssl-sans-tar generate-ssl-wildcard-tar diff --git a/tests/test_coredns_outage b/tests/test_coredns_outage new file mode 100755 index 000000000..6babf416f --- /dev/null +++ b/tests/test_coredns_outage @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -xeo pipefail + +SELF=$(command -v "$0") +TARGET="$1" +SCOPE="${2:-host}" +REMOTE="dokku@$TARGET" +APP="test-coredns-outage-$RANDOM" + +cleanup() { + local exit_code=$? + set +e + start_coredns_docker + ssh "$REMOTE" apps:destroy "$APP" --force >/dev/null 2>&1 + ssh "$REMOTE" events:off >/dev/null 2>&1 + exit $exit_code +} + +stop_coredns_docker() { + case "$SCOPE" in + docker) + docker exec dokku sv stop coredns-docker + ;; + host) + sudo systemctl stop coredns-docker + ;; + *) + echo "Unknown scope: $SCOPE" + exit 1 + ;; + esac +} + +start_coredns_docker() { + case "$SCOPE" in + docker) + docker exec dokku sv start coredns-docker + ;; + host) + sudo systemctl start coredns-docker + ;; + *) + echo "Unknown scope: $SCOPE" + exit 1 + ;; + esac +} + +curl_status() { + curl -kSso /dev/null -w "%{http_code}" -m 5 "http://$APP.$TARGET/" +} + +wait_for_status() { + local expected="$1" attempts="${2:-15}" actual + while [[ $attempts -gt 0 ]]; do + actual="$(curl_status || true)" + [[ "$actual" == "$expected" ]] && return 0 + attempts=$((attempts - 1)) + sleep 1 + done + echo "Expected HTTP $expected, last observed $actual" + return 1 +} + +trap cleanup INT TERM EXIT + +ssh "$REMOTE" events:on + +TMP=$(mktemp -d "/tmp/$TARGET.XXXXX") +rmdir "$TMP" && cp -r "$(dirname "$SELF")/apps/python" "$TMP" + +pushd "$TMP" >/dev/null +git init -q +git config user.email "robot@example.com" +git config user.name "Test Robot" +git remote add target "${REMOTE}:${APP}" +[[ -f gitignore ]] && mv gitignore .gitignore +git add . +git commit -qm 'initial commit' +git push target master +popd >/dev/null +rm -rf "$TMP" + +echo "-----> Verifying app responds before outage" +wait_for_status 200 + +echo "-----> Stopping coredns-docker ($SCOPE)" +stop_coredns_docker + +# Wait long enough for any cached DNS entry (resolver valid=10s in the template) +sleep 12 + +echo "-----> Verifying nginx returns 502 with coredns-docker stopped" +actual="$(curl_status || true)" +if [[ "$actual" != "502" ]]; then + echo "Expected HTTP 502 with coredns-docker stopped, observed $actual" + exit 1 +fi + +echo "-----> Restarting coredns-docker" +start_coredns_docker + +echo "-----> Verifying app recovers" +wait_for_status 200 + +echo "-----> coredns-docker outage test PASSED" diff --git a/tests/unit/nginx-vhosts_17.bats b/tests/unit/nginx-vhosts_17.bats index 01c67e7c8..6f51c6356 100644 --- a/tests/unit/nginx-vhosts_17.bats +++ b/tests/unit/nginx-vhosts_17.bats @@ -9,6 +9,7 @@ setup() { } teardown() { + sudo systemctl start coredns-docker >/dev/null 2>&1 || true dokku --quiet nginx:set --global dns-resolver dokku --quiet nginx:set --global dns-zone destroy_app @@ -137,3 +138,27 @@ teardown() { assert_output_not_contains "\$dokku_upstream" assert_output_contains "upstream ${TEST_APP}-" } + +@test "(nginx-vhosts) nginx returns 502 when coredns-docker is stopped" { + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run sudo systemctl stop coredns-docker + echo "output: $output" + echo "status: $status" + assert_success + + run curl --connect-to "$TEST_APP.$DOKKU_DOMAIN:80:localhost:80" -kSso /dev/null -w "%{http_code}" -m 5 "http://$TEST_APP.$DOKKU_DOMAIN/" + echo "output: $output" + echo "status: $status" + assert_output "502" + + run sudo systemctl start coredns-docker + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response http "$TEST_APP.$DOKKU_DOMAIN" 80 / "" 200 +}