fix: add proxy health checks from app.json readiness

The caddy, haproxy, openresty, and traefik docs stated that requests are only routed to containers passing healthchecks, but these proxies route to a new container as soon as it starts. The first `web` readiness healthcheck in `app.json` is now also turned into health checks and retries for caddy and openresty, as it already was for traefik, so requests are routed away from containers that are not yet ready, and the docs now describe how each proxy actually behaves.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-11-02 00:33:35 -05:00
1 parent 1910c3fb68
commit 83c9c4d72e
15 files changed
+694 -39

No files matched your search

File renamed without changes.
+109 -2
View File
@@ -6,7 +6,8 @@ load test_helper
# Stubs plugn and plugin sources so the trigger can run without a Dokku install.
setup() {
TRIGGER="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)/plugins/caddy-vhosts/docker-args-process-deploy"
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)"
TRIGGER="$REPO_ROOT/plugins/caddy-vhosts/docker-args-process-deploy"
STUB_ROOT="$BATS_TEST_TMPDIR/stub"
mkdir -p "$STUB_ROOT/bin" \
"$STUB_ROOT/core/common" \
@@ -16,10 +17,17 @@ setup() {
cat >"$STUB_ROOT/core/common/functions" <<'EOF'
#!/usr/bin/env bash
dokku_log_warn() { echo "WARN: $*" >&2; }
dokku_log_fail() {
echo "FAIL: $*" >&2
exit 1
}
EOF
cat >"$STUB_ROOT/plugins/proxy/functions" <<'EOF'
cat >"$STUB_ROOT/plugins/proxy/functions" <<EOF
#!/usr/bin/env bash
source "$REPO_ROOT/plugins/proxy/functions"
EOF
cat >>"$STUB_ROOT/plugins/proxy/functions" <<'EOF'
fn-proxy-get-labels-file-path() {
echo "${STUB_LABELS_FILE:-/nonexistent}"
}
@@ -67,6 +75,9 @@ case "$cmd" in
printf '%s\n' "$STUB_DOMAINS"
fi
;;
app-json-get-content)
echo "${STUB_APP_JSON:-{\}}"
;;
*)
echo "unexpected plugn trigger: $cmd $*" >&2
exit 1
@@ -89,6 +100,7 @@ invoke_caddy_docker_args() {
STUB_PROXY_ENABLED="${STUB_PROXY_ENABLED:-true}" \
STUB_VHOST_ENABLED="${STUB_VHOST_ENABLED:-true}" \
STUB_LABELS_FILE="${STUB_LABELS_FILE:-/nonexistent}" \
STUB_APP_JSON="${STUB_APP_JSON:-}" \
bash "$TRIGGER" "${APP_NAME:-testapp}" "dockerfile" "latest" "${PROC_TYPE:-web}" "1" <<<"${STDIN_DATA:-}"
}
@@ -130,3 +142,98 @@ invoke_caddy_docker_args() {
assert_success
assert_output_contains "caddy.reverse_proxy"
}
@test "(caddy-vhosts) readiness healthcheck emits health and retry labels" {
STUB_DOMAINS="example.com"
STUB_PORTS="http:80:5000"
STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/health","timeout":3,"wait":2}]}}'
run invoke_caddy_docker_args
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains '--label "caddy.reverse_proxy.health_uri=/health"'
assert_output_contains "--label caddy.reverse_proxy.health_interval=2s"
assert_output_contains "--label caddy.reverse_proxy.health_timeout=3s"
assert_output_contains "--label caddy.reverse_proxy.health_headers.Host=example.com"
assert_output_contains "--label caddy.reverse_proxy.lb_try_duration=5s"
assert_output_contains "--label caddy.reverse_proxy.lb_try_interval=250ms"
assert_output_contains "health_port" 0
}
@test "(caddy-vhosts) readiness healthcheck defaults and port" {
STUB_DOMAINS="example.com"
STUB_PORTS="http:80:5000"
STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","port":5001}]}}'
run invoke_caddy_docker_args
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "--label caddy.reverse_proxy.health_port=5001"
assert_output_contains "--label caddy.reverse_proxy.health_interval=5s"
assert_output_contains "--label caddy.reverse_proxy.health_timeout=5s"
}
@test "(caddy-vhosts) healthcheck host header skips localhost and catch-all domains" {
STUB_DOMAINS="localhost * app.example.com other.example.com"
STUB_PORTS="http:80:5000"
STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/"}]}}'
run invoke_caddy_docker_args
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "--label caddy.reverse_proxy.health_headers.Host=app.example.com"
}
@test "(caddy-vhosts) startup healthcheck emits no health or retry labels" {
STUB_DOMAINS="example.com"
STUB_PORTS="http:80:5000"
STUB_APP_JSON='{"healthchecks":{"web":[{"type":"startup","path":"/"}]}}'
run invoke_caddy_docker_args
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "caddy.reverse_proxy="
assert_output_contains "health_" 0
assert_output_contains "lb_try_" 0
}
@test "(caddy-vhosts) missing app.json emits no health or retry labels" {
STUB_DOMAINS="example.com"
STUB_PORTS="http:80:5000"
run invoke_caddy_docker_args
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "caddy.reverse_proxy="
assert_output_contains "health_" 0
assert_output_contains "lb_try_" 0
}
@test "(caddy-vhosts) readiness healthcheck without routing labels emits no health labels" {
STUB_DOMAINS="example.com"
STUB_PORTS=""
STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/"}]}}'
run invoke_caddy_docker_args
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "health_" 0
}
@test "(caddy-vhosts) invalid readiness healthcheck path fails" {
STUB_DOMAINS="example.com"
STUB_PORTS="http:80:5000"
STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/has space"}]}}'
run invoke_caddy_docker_args
echo "output: $output"
echo "status: $status"
assert_failure
assert_output_contains "Invalid app.json healthcheck path for caddy"
}
+139
View File
@@ -452,3 +452,142 @@ teardown() {
echo "status: $status"
assert_output "0"
}
@test "(caddy) healthcheck labels from app.json" {
run /bin/bash -c "dokku proxy:set $TEST_APP caddy"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_uri\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "/"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_interval\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "2s"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_timeout\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "5s"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_headers.Host\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "$TEST_APP.$DOKKU_DOMAIN"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_duration\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "5s"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_interval\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "250ms"
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
}
@test "(caddy) healthcheck labels can be overridden" {
run /bin/bash -c "dokku proxy:set $TEST_APP caddy"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku caddy:labels:add $TEST_APP caddy.reverse_proxy.lb_try_duration 10s"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_duration\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "10s"
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
}
@test "(caddy) no healthcheck labels without readiness check" {
run /bin/bash -c "dokku proxy:set $TEST_APP caddy"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_uri\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_not_exists
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_duration\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_not_exists
}
@test "(caddy) invalid stored healthcheck path fails restart" {
run /bin/bash -c "dokku proxy:set $TEST_APP caddy"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "jq '.healthchecks.web[0].path = \"/health check\"' /var/lib/dokku/data/app-json/$TEST_APP/app.json > /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp && mv /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp /var/lib/dokku/data/app-json/$TEST_APP/app.json"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:restart $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
assert_output_contains "Invalid app.json healthcheck path for caddy" -1
}
setup_caddy_readiness_healthcheck() {
local APP="$1"
local APP_REPO_DIR="$2"
[[ -z "$APP" ]] && local APP="$TEST_APP"
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
convert_to_dockerfile "$APP" "$APP_REPO_DIR"
mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json"
}
+189
View File
@@ -760,6 +760,175 @@ teardown() {
assert_output_not_exists
}
@test "(openresty) healthcheck labels from app.json" {
run /bin/bash -c "dokku proxy:set $TEST_APP openresty"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_openresty_readiness_healthcheck
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.upstream-max-fails\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "1"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.upstream-fail-timeout\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "2s"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.proxy-next-upstream-timeout\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "5s"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-path\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "/"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-interval\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "2"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-timeout\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "5"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-fall\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "3"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-host\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "$TEST_APP.dokku.me"
run /bin/bash -c "docker exec openresty-openresty-1 /usr/local/openresty/nginx/sbin/nginx -t"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker exec openresty-openresty-1 cat /etc/nginx/sites-enabled/sites.conf"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "max_fails=1 fail_timeout=2s;"
assert_output_contains "proxy_next_upstream_timeout 5s;"
sleep 5
run /bin/bash -c "docker exec openresty-openresty-1 wget -qO- http://127.0.0.1:8999/upstream-healthcheck-status"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "Upstream $TEST_APP-web-5000"
assert_output_contains " UP"
assert_output_contains "(NO checkers)" 0
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
}
@test "(openresty) no healthcheck labels without readiness check" {
run /bin/bash -c "dokku proxy:set $TEST_APP openresty"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.upstream-max-fails\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_not_exists
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.proxy-next-upstream-timeout\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_not_exists
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-path\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_not_exists
}
@test "(openresty) healthcheck headers from app.json" {
run /bin/bash -c "dokku proxy:set $TEST_APP openresty"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_openresty_readiness_healthcheck_headers
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-header.X-Check\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "dokku"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-host\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "check.dokku.me"
run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-header.Host\" }}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_not_exists
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
}
@test "(openresty) invalid stored healthcheck path fails restart" {
run /bin/bash -c "dokku proxy:set $TEST_APP openresty"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_openresty_readiness_healthcheck
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "jq '.healthchecks.web[0].path = \"/health check\"' /var/lib/dokku/data/app-json/$TEST_APP/app.json > /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp && mv /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp /var/lib/dokku/data/app-json/$TEST_APP/app.json"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:restart $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_failure
assert_output_contains "Invalid app.json healthcheck path for openresty" -1
}
add_openresty_include() {
local APP="$1"
local APP_REPO_DIR="$2"
@@ -786,3 +955,23 @@ add_openresty_include_unsafe() {
mkdir -p "$APP_REPO_DIR/openresty/http-location-includes"
printf '# location\n' >"$APP_REPO_DIR/openresty/http-location-includes/unsafe filename.conf"
}
setup_openresty_readiness_healthcheck() {
local APP="$1"
local APP_REPO_DIR="$2"
[[ -z "$APP" ]] && local APP="$TEST_APP"
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
convert_to_dockerfile "$APP" "$APP_REPO_DIR"
mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json"
}
setup_openresty_readiness_healthcheck_headers() {
local APP="$1"
local APP_REPO_DIR="$2"
[[ -z "$APP" ]] && local APP="$TEST_APP"
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
convert_to_dockerfile "$APP" "$APP_REPO_DIR"
jq '.healthchecks.web[0].httpHeaders = [{"name": "X-Check", "value": "dokku"}, {"name": "Host", "value": "check.dokku.me"}]' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json"
}
+3 -3
View File
@@ -943,7 +943,7 @@ setup_traefik_healthcheck() {
[[ -z "$APP" ]] && local APP="$TEST_APP"
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
mv "$APP_REPO_DIR/app-traefik.json" "$APP_REPO_DIR/app.json"
mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json"
}
setup_traefik_healthcheck_whitespace_path() {
@@ -952,7 +952,7 @@ setup_traefik_healthcheck_whitespace_path() {
[[ -z "$APP" ]] && local APP="$TEST_APP"
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
jq '.healthchecks.web[0].path = "/health check"' "$APP_REPO_DIR/app-traefik.json" >"$APP_REPO_DIR/app.json"
jq '.healthchecks.web[0].path = "/health check"' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json"
}
setup_traefik_healthcheck_invalid_scheme() {
@@ -961,5 +961,5 @@ setup_traefik_healthcheck_invalid_scheme() {
[[ -z "$APP" ]] && local APP="$TEST_APP"
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
jq '.healthchecks.web[0].scheme = "tcp"' "$APP_REPO_DIR/app-traefik.json" >"$APP_REPO_DIR/app.json"
jq '.healthchecks.web[0].scheme = "tcp"' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json"
}