From 83c9c4d72e2a503fd5ff0d8b855502247954e5f3 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 26 Sep 2026 01:01:41 -0400 Subject: [PATCH 1/4] fix: add proxy health checks from app.json readiness The caddy, haproxy, openresty, and traefik docs stated that requests are only routed to containers passing healthchecks, but these proxies route to a new container as soon as it starts. The first `web` readiness healthcheck in `app.json` is now also turned into health checks and retries for caddy and openresty, as it already was for traefik, so requests are routed away from containers that are not yet ready, and the docs now describe how each proxy actually behaves. --- docs/deployment/zero-downtime-deploys.md | 12 ++ docs/networking/proxies/caddy.md | 48 ++++- docs/networking/proxies/haproxy.md | 2 +- docs/networking/proxies/openresty.md | 39 +++- docs/networking/proxies/traefik.md | 6 +- .../caddy-vhosts/docker-args-process-deploy | 43 ++++ .../docker-args-process-deploy | 74 +++++++ plugins/proxy/functions | 26 +++ .../traefik-vhosts/docker-args-process-deploy | 21 +- plugins/traefik-vhosts/internal-functions | 17 -- .../{app-traefik.json => app-readiness.json} | 0 tests/unit/caddy-vhosts-docker-args.bats | 111 +++++++++- tests/unit/caddy.bats | 139 +++++++++++++ tests/unit/openresty.bats | 189 ++++++++++++++++++ tests/unit/traefik.bats | 6 +- 15 files changed, 694 insertions(+), 39 deletions(-) rename tests/apps/python/{app-traefik.json => app-readiness.json} (100%) diff --git a/docs/deployment/zero-downtime-deploys.md b/docs/deployment/zero-downtime-deploys.md index c737f56e1..f7a9a9a02 100644 --- a/docs/deployment/zero-downtime-deploys.md +++ b/docs/deployment/zero-downtime-deploys.md @@ -180,6 +180,18 @@ A healthcheck entry takes the following properties: See the [docker-container-healthchecker](https://github.com/dokku/docker-container-healthchecker) documentation for more details on how healthchecks are interpreted. +### Healthchecks and label-based proxies + +When using the `docker-local` scheduler, the `nginx` proxy only routes traffic to new containers once healthchecks have passed. The `caddy`, `haproxy`, `openresty`, and `traefik` proxies instead discover containers via docker labels, which are attached when a container is created, and will route requests to a new container as soon as it is running. As Dokku's healthchecks are run from the host, these proxies are not aware of their result. + +To reduce the number of requests routed to a container that is not yet ready, the first `readiness` healthcheck with a `path` for the `web` process is translated into proxy-specific configuration: + +- [Caddy](/docs/networking/proxies/caddy.md#healthchecks): Active healthchecks against the `path`, and retries of failed requests against other containers. +- [OpenResty](/docs/networking/proxies/openresty.md#healthchecks): Active healthchecks against the `path`, and retries of failed requests against other containers. +- [Traefik](/docs/networking/proxies/traefik.md#healthchecks): Active healthchecks against the `path`. + +Haproxy does not support this configuration. Healthchecks of other types are not translated, as proxy healthchecks run for the lifetime of a container. Translating a `startup` healthcheck would turn it into a `readiness` healthcheck. + ## Manually invoking checks Checks can also be manually invoked via the `checks:run` command. This can be used to check the status of an application via cron to provide integration with external healthchecking software. diff --git a/docs/networking/proxies/caddy.md b/docs/networking/proxies/caddy.md index 490eb7dac..3484e8e57 100644 --- a/docs/networking/proxies/caddy.md +++ b/docs/networking/proxies/caddy.md @@ -32,7 +32,53 @@ The Caddy plugin has specific rules for routing requests: - If no `http:80` mapping is found, the first `http` port mapping is used for http requests. - If no `https:443` mapping is found, the first `https` port mapping is used for https requests. - If no `https` mapping is found, the container port from `http:80` will be used for https requests. -- Requests are routed as soon as the container is running and passing healthchecks. +- Requests are routed as soon as the container is running, before Dokku's healthchecks have passed. See the [healthchecks](#healthchecks) section for how Caddy avoids routing to containers that are not yet ready. + +### Healthchecks + +When an app has a readiness healthcheck defined in its `app.json` file with a `path` property, Dokku automatically generates Caddy healthcheck and retry labels. These configure Caddy to periodically check each container, stop routing traffic to containers that fail the check, and retry requests that fail to connect against another container. + +The following `app.json` healthcheck properties are mapped to Caddy labels: + +| app.json Property | Caddy Label | Description | +|-------------------|-------------|-------------| +| `path` | `caddy.reverse_proxy.health_uri` | The HTTP path to check (required) | +| `port` | `caddy.reverse_proxy.health_port` | The port to check | +| `timeout` | `caddy.reverse_proxy.health_timeout` | Timeout in seconds (formatted as `Xs`, defaults to `5s`) | +| `wait` | `caddy.reverse_proxy.health_interval` | Interval between checks in seconds (formatted as `Xs`, defaults to `5s`) | + +Additionally, the following labels are generated: + +| Caddy Label | Value | Description | +|-------------|-------|-------------| +| `caddy.reverse_proxy.health_headers.Host` | the first app domain | The `Host` header sent with the healthcheck, matching the header Dokku sends during deploy checks | +| `caddy.reverse_proxy.lb_try_duration` | `5s` | How long a request that fails to connect is retried against other containers | +| `caddy.reverse_proxy.lb_try_interval` | `250ms` | How long to wait between retries | + +Example `app.json` configuration: + +```json +{ + "healthchecks": { + "web": [ + { + "name": "web readiness check", + "path": "/health", + "timeout": 5, + "type": "readiness", + "wait": 10 + } + ] + } +} +``` + +Caddy considers a new container healthy until its first check fails. The first check runs as soon as Caddy picks up the new container, and requests that reach the container before then and fail to connect are retried against the still-running old container. + +The `scheme` property is not mapped, as Caddy uses the same transport for healthchecks as for proxied requests. Any of the generated labels may be overridden via the `caddy:labels:add` command. + +> [!NOTE] +> Only the first readiness healthcheck with a `path` property is used. Apps without one are routed as soon as the container is running. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#healthchecks-and-label-based-proxies) for more information. ### Switching to Caddy diff --git a/docs/networking/proxies/haproxy.md b/docs/networking/proxies/haproxy.md index b85980d7e..93afa317f 100644 --- a/docs/networking/proxies/haproxy.md +++ b/docs/networking/proxies/haproxy.md @@ -28,7 +28,7 @@ The Haproxy plugin has specific rules for routing requests: - Haproxy integration is exposed via docker labels attached to containers. Changes in labels require either app deploys or rebuilds. - While Haproxy will respect labels associated with other containers, only `web` containers have Haproxy labels injected by the plugin. - Only `http:80` and `https:443` port mappings are supported at this time. -- Requests are routed as soon as the container is running and passing healthchecks. +- Requests are routed as soon as the container is running, before Dokku's healthchecks have passed. Haproxy does not support healthcheck configuration via container labels, so `app.json` healthchecks are not used for routing. ### Switching to Haproxy diff --git a/docs/networking/proxies/openresty.md b/docs/networking/proxies/openresty.md index 274dabd4b..3c62eb9fe 100644 --- a/docs/networking/proxies/openresty.md +++ b/docs/networking/proxies/openresty.md @@ -28,7 +28,44 @@ The OpenResty plugin has specific rules for routing requests: - OpenResty integration is exposed via docker labels attached to containers. Changes in labels require either app deploys or rebuilds. - While OpenResty will respect labels associated with other containers, only `web` containers have OpenResty labels injected by the plugin. - Only `http:80` and `https:443` port mappings are supported at this time. -- Requests are routed as soon as the container is running and passing healthchecks. +- Requests are routed as soon as the container is running, before Dokku's healthchecks have passed. See the [healthchecks](#healthchecks) section for how failed requests to a container that is not yet listening are handled. + +### Healthchecks + +When an app has a readiness healthcheck defined in its `app.json` file with a `path` property, Dokku automatically generates OpenResty healthcheck labels. These configure OpenResty to periodically check each container and stop routing traffic to containers that fail the check until they pass again. + +The following `app.json` healthcheck properties are mapped to OpenResty labels: + +| app.json Property | OpenResty Label | Description | +|-------------------|-----------------|-------------| +| `path` | `openresty.healthcheck-path` | The HTTP path to check (required) | +| `port` | `openresty.healthcheck-port` | The port to check | +| `scheme` | `openresty.healthcheck-scheme` | The scheme to use (`http` or `https`) | +| `timeout` | `openresty.healthcheck-timeout` | Timeout in seconds (defaults to `5`) | +| `wait` | `openresty.healthcheck-interval` | Interval between checks in seconds (defaults to `5`) | +| `attempts` | `openresty.healthcheck-fall` | Number of successive failed checks before a container is taken out of rotation (defaults to `3`) | +| `httpHeaders` | `openresty.healthcheck-header.` | Headers to send with the check. A `Host` header is set via `openresty.healthcheck-host` instead | + +If no `Host` header is specified, the first app domain is used, matching the header Dokku sends during deploy checks. Headers with invalid names or values containing quotes, backslashes, or control characters are skipped. + +Additionally, the following labels are generated so that a request that fails to connect to a container is retried against another container, and the failing container is immediately taken out of rotation: + +| OpenResty Label | Value | Description | +|-----------------|-------|-------------| +| `openresty.upstream-max-fails` | `1` | Number of failed requests before a container is taken out of rotation | +| `openresty.upstream-fail-timeout` | the healthcheck `wait` value, or `5s` | How long a container is taken out of rotation after a failed request | +| `openresty.proxy-next-upstream-timeout` | `5s` | How long a failed request may be retried against other containers | + +OpenResty considers a new container healthy until it fails a check or a request to it fails to connect, and healthcheck state is reset whenever the OpenResty configuration is reloaded. Responses with a `2xx` or `3xx` status are considered healthy, and the `content` property is not supported. + +The status of all checked upstreams can be viewed from within the OpenResty container: + +```shell +docker exec openresty-openresty-1 wget -qO- http://127.0.0.1:8999/upstream-healthcheck-status +``` + +> [!NOTE] +> Only the first readiness healthcheck with a `path` property is used. Apps without one are routed as soon as the container is running, though requests that fail to connect to a container are still retried against other containers. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#healthchecks-and-label-based-proxies) for more information. ### Switching to OpenResty diff --git a/docs/networking/proxies/traefik.md b/docs/networking/proxies/traefik.md index 2e7a08fbe..425760152 100644 --- a/docs/networking/proxies/traefik.md +++ b/docs/networking/proxies/traefik.md @@ -31,12 +31,12 @@ The Traefik plugin has specific rules for routing requests: - If no `http:80` mapping is found, the first `http` port mapping is used for http requests. - If no `https:443` mapping is found, the first `https` port mapping is used for https requests. - If no `https` mapping is found, the container port from `http:80` will be used for https requests. -- Requests are routed as soon as the container is running and passing healthchecks. +- Requests are routed as soon as the container is running, before Dokku's healthchecks have passed. - Readiness healthchecks defined in `app.json` with a `path` property are automatically transformed into Traefik healthcheck labels. ### Healthchecks -When an app has a readiness healthcheck defined in its `app.json` file with a `path` property, Dokku automatically generates Traefik healthcheck labels. These labels configure Traefik to perform health checks on the container before routing traffic to it. +When an app has a readiness healthcheck defined in its `app.json` file with a `path` property, Dokku automatically generates Traefik healthcheck labels. These labels configure Traefik to periodically check each container and stop routing traffic to containers that fail the check. Traefik considers a new container healthy until its first check fails, so requests may reach a container before its first check runs. The following `app.json` healthcheck properties are mapped to Traefik labels: @@ -67,7 +67,7 @@ Example `app.json` configuration: ``` > [!NOTE] -> Only the first readiness healthcheck with a `path` property is used. Liveness, startup, and command-based healthchecks are not transformed into Traefik labels. +> Only the first readiness healthcheck with a `path` property is used. Liveness, startup, and command-based healthchecks are not transformed into Traefik labels. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#healthchecks-and-label-based-proxies) for more information. ### Switching to Traefik diff --git a/plugins/caddy-vhosts/docker-args-process-deploy b/plugins/caddy-vhosts/docker-args-process-deploy index 184de7149..5235c2b67 100755 --- a/plugins/caddy-vhosts/docker-args-process-deploy +++ b/plugins/caddy-vhosts/docker-args-process-deploy @@ -127,6 +127,10 @@ trigger-caddy-vhosts-docker-args-process-deploy() { output="$output --label \"caddy.reverse_proxy={{ upstreams $proxy_container_http_port }}\"" fi + + if [[ "$output" == *"caddy.reverse_proxy="* ]]; then + output="$output$(fn-caddy-healthcheck-labels "$APP" "$app_domains")" + fi fi local proxy_labels_file_path=$(fn-proxy-get-labels-file-path "caddy" "$APP") @@ -140,4 +144,43 @@ trigger-caddy-vhosts-docker-args-process-deploy() { echo -n "$STDIN $output" } +fn-caddy-healthcheck-labels() { + declare desc="outputs caddy reverse_proxy health check and retry labels derived from the app's web readiness healthcheck" + declare APP="$1" APP_DOMAINS="$2" + local healthcheck hc_path hc_port hc_timeout hc_interval hc_host domain + local -a domains + + healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")" + if [[ -z "$healthcheck" ]]; then + return + fi + + hc_path="$(echo "$healthcheck" | jq -r ".path // empty")" + hc_port="$(echo "$healthcheck" | jq -r ".port // empty")" + hc_timeout="$(echo "$healthcheck" | jq -r ".timeout // 5")" + hc_interval="$(echo "$healthcheck" | jq -r ".wait // 5")" + + if ! fn-proxy-healthcheck-path-is-valid "$hc_path"; then + dokku_log_fail "Invalid app.json healthcheck path for caddy: path must start with a / and must not contain whitespace, control characters, quotes, or backslashes" + fi + + read -r -d '' -a domains <<<"$APP_DOMAINS" || true + for domain in "${domains[@]}"; do + if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then + continue + fi + + hc_host="$domain" + break + done + + echo -n " --label \"caddy.reverse_proxy.health_uri=$hc_path\"" + [[ -n "$hc_port" ]] && echo -n " --label caddy.reverse_proxy.health_port=$hc_port" + echo -n " --label caddy.reverse_proxy.health_interval=${hc_interval}s" + echo -n " --label caddy.reverse_proxy.health_timeout=${hc_timeout}s" + [[ -n "$hc_host" ]] && echo -n " --label caddy.reverse_proxy.health_headers.Host=$hc_host" + echo -n " --label caddy.reverse_proxy.lb_try_duration=5s" + echo -n " --label caddy.reverse_proxy.lb_try_interval=250ms" +} + trigger-caddy-vhosts-docker-args-process-deploy "$@" diff --git a/plugins/openresty-vhosts/docker-args-process-deploy b/plugins/openresty-vhosts/docker-args-process-deploy index 3488f4c52..9bbe1fb16 100755 --- a/plugins/openresty-vhosts/docker-args-process-deploy +++ b/plugins/openresty-vhosts/docker-args-process-deploy @@ -206,6 +206,8 @@ trigger-openresty-vhosts-docker-args-process-deploy() { value="$(fn-openresty-computed-x-forwarded-ssl "$APP")" [[ -n "$value" ]] && output="$output '--label=openresty.x-forwarded-ssl=$value'" + output="$output$(fn-openresty-healthcheck-labels "$APP")" + local proxy_labels_file_path=$(fn-proxy-get-labels-file-path "openresty" "$APP") if [[ -f "$proxy_labels_file_path" ]]; then while read -r line; do @@ -217,4 +219,76 @@ trigger-openresty-vhosts-docker-args-process-deploy() { echo -n "$STDIN $output" } +fn-openresty-healthcheck-labels() { + declare desc="outputs openresty upstream healthcheck and failover labels derived from the app's web readiness healthcheck" + declare APP="$1" + local healthcheck hc_path hc_port hc_scheme hc_timeout hc_interval hc_attempts hc_host domain name value + local -a domains + + healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")" + if [[ -z "$healthcheck" ]]; then + return + fi + + hc_path="$(echo "$healthcheck" | jq -r ".path // empty")" + hc_port="$(echo "$healthcheck" | jq -r ".port // empty")" + hc_scheme="$(echo "$healthcheck" | jq -r ".scheme // empty")" + hc_timeout="$(echo "$healthcheck" | jq -r ".timeout // 5")" + hc_interval="$(echo "$healthcheck" | jq -r ".wait // 5")" + hc_attempts="$(echo "$healthcheck" | jq -r ".attempts // 3")" + + if ! fn-proxy-healthcheck-path-is-valid "$hc_path"; then + dokku_log_fail "Invalid app.json healthcheck path for openresty: path must start with a / and must not contain whitespace, control characters, quotes, or backslashes" + fi + + if ! fn-proxy-healthcheck-scheme-is-valid "$hc_scheme"; then + dokku_log_fail "Invalid app.json healthcheck scheme for openresty: scheme must be one of http or https" + fi + + hc_host="$(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) == "host")) | .[0].value // empty')" + if [[ -z "$hc_host" ]]; then + read -r -d '' -a domains <<<"$(plugn trigger domains-list "$APP")" || true + for domain in "${domains[@]}"; do + if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then + continue + fi + + hc_host="$domain" + break + done + fi + + echo -n " '--label=openresty.upstream-max-fails=1'" + echo -n " '--label=openresty.upstream-fail-timeout=${hc_interval}s'" + echo -n " '--label=openresty.proxy-next-upstream-timeout=5s'" + echo -n " '--label=openresty.healthcheck-path=$hc_path'" + [[ -n "$hc_port" ]] && echo -n " '--label=openresty.healthcheck-port=$hc_port'" + [[ -n "$hc_scheme" ]] && echo -n " '--label=openresty.healthcheck-scheme=${hc_scheme,,}'" + echo -n " '--label=openresty.healthcheck-timeout=$hc_timeout'" + echo -n " '--label=openresty.healthcheck-interval=$hc_interval'" + echo -n " '--label=openresty.healthcheck-fall=$hc_attempts'" + if [[ -n "$hc_host" ]] && fn-openresty-healthcheck-header-is-valid "Host" "$hc_host"; then + echo -n " '--label=openresty.healthcheck-host=$hc_host'" + fi + + while IFS=$'\t' read -r name value; do + [[ -z "$name" ]] && continue + if ! fn-openresty-healthcheck-header-is-valid "$name" "$value"; then + dokku_log_warn "Skipping invalid app.json healthcheck header for openresty: $name" + continue + fi + + echo -n " '--label=openresty.healthcheck-header.$name=$value'" + done < <(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) != "host")) | .[] | [.name, .value] | @tsv') +} + +fn-openresty-healthcheck-header-is-valid() { + declare desc="checks that a healthcheck header name is a token and its value is free of control, quote, and backslash characters" + declare NAME="$1" VALUE="$2" + + [[ "$NAME" =~ ^[A-Za-z0-9!#$%\&*+.^_\`\|~-]+$ ]] || return 1 + [[ "$VALUE" =~ [[:cntrl:]\"\'\\] ]] && return 1 + return 0 +} + trigger-openresty-vhosts-docker-args-process-deploy "$@" diff --git a/plugins/proxy/functions b/plugins/proxy/functions index c51192e98..46e994db3 100755 --- a/plugins/proxy/functions +++ b/plugins/proxy/functions @@ -17,6 +17,32 @@ fn-proxy-get-labels-file-path() { echo "$PROXY_FILE" } +fn-proxy-get-readiness-healthcheck() { + declare desc="outputs the first web readiness healthcheck with a path from the app's app.json as json" + declare APP="$1" + local app_json + + app_json="$(plugn trigger app-json-get-content "$APP" 2>/dev/null || echo "{}")" + echo "$app_json" | jq -c '.healthchecks.web // [] | map(select(.type == "readiness" and .path != null and .path != "")) | .[0] // empty' +} + +fn-proxy-healthcheck-path-is-valid() { + declare desc="checks that an app.json healthcheck path is absolute and free of whitespace, control, quote, and backslash characters" + declare HC_PATH="$1" + + [[ "$HC_PATH" == /* ]] || return 1 + [[ "$HC_PATH" =~ [[:space:][:cntrl:]\"\'\\] ]] && return 1 + return 0 +} + +fn-proxy-healthcheck-scheme-is-valid() { + declare desc="checks that an app.json healthcheck scheme is either http or https" + declare HC_SCHEME="$1" + + [[ -z "$HC_SCHEME" ]] && return 0 + [[ "${HC_SCHEME,,}" == "http" ]] || [[ "${HC_SCHEME,,}" == "https" ]] +} + fn-proxy-display-labels() { declare desc="print user-set app container labels for specified proxy" declare PROXY="$1" APP="$2" diff --git a/plugins/traefik-vhosts/docker-args-process-deploy b/plugins/traefik-vhosts/docker-args-process-deploy index 29f8a6e45..200dab355 100755 --- a/plugins/traefik-vhosts/docker-args-process-deploy +++ b/plugins/traefik-vhosts/docker-args-process-deploy @@ -130,21 +130,20 @@ trigger-traefik-vhosts-docker-args-process-deploy() { fi fi - local app_json hc_path hc_scheme hc_port hc_timeout hc_interval hc_filter - app_json="$(plugn trigger app-json-get-content "$APP" 2>/dev/null || echo "{}")" - hc_filter='.healthchecks.web // [] | map(select(.type == "readiness" and .path != null and .path != "")) | .[0]' - hc_path=$(echo "$app_json" | jq -r "$hc_filter | .path // empty") - if [[ -n "$hc_path" ]]; then - hc_scheme=$(echo "$app_json" | jq -r "$hc_filter | .scheme // empty") - hc_port=$(echo "$app_json" | jq -r "$hc_filter | .port // empty") - hc_timeout=$(echo "$app_json" | jq -r "$hc_filter | .timeout // empty") - hc_interval=$(echo "$app_json" | jq -r "$hc_filter | .wait // empty") + local healthcheck hc_path hc_scheme hc_port hc_timeout hc_interval + healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")" + if [[ -n "$healthcheck" ]]; then + hc_path=$(echo "$healthcheck" | jq -r ".path // empty") + hc_scheme=$(echo "$healthcheck" | jq -r ".scheme // empty") + hc_port=$(echo "$healthcheck" | jq -r ".port // empty") + hc_timeout=$(echo "$healthcheck" | jq -r ".timeout // empty") + hc_interval=$(echo "$healthcheck" | jq -r ".wait // empty") - if ! fn-traefik-healthcheck-path-is-valid "$hc_path"; then + if ! fn-proxy-healthcheck-path-is-valid "$hc_path"; then dokku_log_fail "Invalid app.json healthcheck path for traefik: path must start with a / and must not contain whitespace, control characters, quotes, or backslashes" fi - if ! fn-traefik-healthcheck-scheme-is-valid "$hc_scheme"; then + if ! fn-proxy-healthcheck-scheme-is-valid "$hc_scheme"; then dokku_log_fail "Invalid app.json healthcheck scheme for traefik: scheme must be one of http or https" fi diff --git a/plugins/traefik-vhosts/internal-functions b/plugins/traefik-vhosts/internal-functions index de94e656d..49b9afc5c 100755 --- a/plugins/traefik-vhosts/internal-functions +++ b/plugins/traefik-vhosts/internal-functions @@ -243,20 +243,3 @@ fn-traefik-dns-provider-env-vars() { declare desc="returns all dns-provider-* environment variables" fn-plugin-property-get-all-by-prefix "traefik" "--global" "dns-provider-" } - -fn-traefik-healthcheck-path-is-valid() { - declare desc="checks that an app.json healthcheck path is absolute and free of whitespace, control, quote, and backslash characters" - declare HC_PATH="$1" - - [[ "$HC_PATH" == /* ]] || return 1 - [[ "$HC_PATH" =~ [[:space:][:cntrl:]\"\'\\] ]] && return 1 - return 0 -} - -fn-traefik-healthcheck-scheme-is-valid() { - declare desc="checks that an app.json healthcheck scheme is either http or https" - declare HC_SCHEME="$1" - - [[ -z "$HC_SCHEME" ]] && return 0 - [[ "${HC_SCHEME,,}" == "http" ]] || [[ "${HC_SCHEME,,}" == "https" ]] -} diff --git a/tests/apps/python/app-traefik.json b/tests/apps/python/app-readiness.json similarity index 100% rename from tests/apps/python/app-traefik.json rename to tests/apps/python/app-readiness.json diff --git a/tests/unit/caddy-vhosts-docker-args.bats b/tests/unit/caddy-vhosts-docker-args.bats index 032cf4943..223b648ce 100644 --- a/tests/unit/caddy-vhosts-docker-args.bats +++ b/tests/unit/caddy-vhosts-docker-args.bats @@ -6,7 +6,8 @@ load test_helper # Stubs plugn and plugin sources so the trigger can run without a Dokku install. setup() { - TRIGGER="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)/plugins/caddy-vhosts/docker-args-process-deploy" + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + TRIGGER="$REPO_ROOT/plugins/caddy-vhosts/docker-args-process-deploy" STUB_ROOT="$BATS_TEST_TMPDIR/stub" mkdir -p "$STUB_ROOT/bin" \ "$STUB_ROOT/core/common" \ @@ -16,10 +17,17 @@ setup() { cat >"$STUB_ROOT/core/common/functions" <<'EOF' #!/usr/bin/env bash dokku_log_warn() { echo "WARN: $*" >&2; } +dokku_log_fail() { + echo "FAIL: $*" >&2 + exit 1 +} EOF - cat >"$STUB_ROOT/plugins/proxy/functions" <<'EOF' + cat >"$STUB_ROOT/plugins/proxy/functions" <>"$STUB_ROOT/plugins/proxy/functions" <<'EOF' fn-proxy-get-labels-file-path() { echo "${STUB_LABELS_FILE:-/nonexistent}" } @@ -67,6 +75,9 @@ case "$cmd" in printf '%s\n' "$STUB_DOMAINS" fi ;; + app-json-get-content) + echo "${STUB_APP_JSON:-{\}}" + ;; *) echo "unexpected plugn trigger: $cmd $*" >&2 exit 1 @@ -89,6 +100,7 @@ invoke_caddy_docker_args() { STUB_PROXY_ENABLED="${STUB_PROXY_ENABLED:-true}" \ STUB_VHOST_ENABLED="${STUB_VHOST_ENABLED:-true}" \ STUB_LABELS_FILE="${STUB_LABELS_FILE:-/nonexistent}" \ + STUB_APP_JSON="${STUB_APP_JSON:-}" \ bash "$TRIGGER" "${APP_NAME:-testapp}" "dockerfile" "latest" "${PROC_TYPE:-web}" "1" <<<"${STDIN_DATA:-}" } @@ -130,3 +142,98 @@ invoke_caddy_docker_args() { assert_success assert_output_contains "caddy.reverse_proxy" } + +@test "(caddy-vhosts) readiness healthcheck emits health and retry labels" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/health","timeout":3,"wait":2}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains '--label "caddy.reverse_proxy.health_uri=/health"' + assert_output_contains "--label caddy.reverse_proxy.health_interval=2s" + assert_output_contains "--label caddy.reverse_proxy.health_timeout=3s" + assert_output_contains "--label caddy.reverse_proxy.health_headers.Host=example.com" + assert_output_contains "--label caddy.reverse_proxy.lb_try_duration=5s" + assert_output_contains "--label caddy.reverse_proxy.lb_try_interval=250ms" + assert_output_contains "health_port" 0 +} + +@test "(caddy-vhosts) readiness healthcheck defaults and port" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","port":5001}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label caddy.reverse_proxy.health_port=5001" + assert_output_contains "--label caddy.reverse_proxy.health_interval=5s" + assert_output_contains "--label caddy.reverse_proxy.health_timeout=5s" +} + +@test "(caddy-vhosts) healthcheck host header skips localhost and catch-all domains" { + STUB_DOMAINS="localhost * app.example.com other.example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/"}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label caddy.reverse_proxy.health_headers.Host=app.example.com" +} + +@test "(caddy-vhosts) startup healthcheck emits no health or retry labels" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"startup","path":"/"}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "caddy.reverse_proxy=" + assert_output_contains "health_" 0 + assert_output_contains "lb_try_" 0 +} + +@test "(caddy-vhosts) missing app.json emits no health or retry labels" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "caddy.reverse_proxy=" + assert_output_contains "health_" 0 + assert_output_contains "lb_try_" 0 +} + +@test "(caddy-vhosts) readiness healthcheck without routing labels emits no health labels" { + STUB_DOMAINS="example.com" + STUB_PORTS="" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/"}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "health_" 0 +} + +@test "(caddy-vhosts) invalid readiness healthcheck path fails" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/has space"}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_failure + assert_output_contains "Invalid app.json healthcheck path for caddy" +} diff --git a/tests/unit/caddy.bats b/tests/unit/caddy.bats index e78a160a4..5d8342bb6 100644 --- a/tests/unit/caddy.bats +++ b/tests/unit/caddy.bats @@ -452,3 +452,142 @@ teardown() { echo "status: $status" assert_output "0" } + +@test "(caddy) healthcheck labels from app.json" { + run /bin/bash -c "dokku proxy:set $TEST_APP caddy" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_uri\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "/" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_interval\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "2s" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_timeout\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "5s" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_headers.Host\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$TEST_APP.$DOKKU_DOMAIN" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_duration\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "5s" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_interval\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "250ms" + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + +@test "(caddy) healthcheck labels can be overridden" { + run /bin/bash -c "dokku proxy:set $TEST_APP caddy" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku caddy:labels:add $TEST_APP caddy.reverse_proxy.lb_try_duration 10s" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_duration\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "10s" + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + +@test "(caddy) no healthcheck labels without readiness check" { + run /bin/bash -c "dokku proxy:set $TEST_APP caddy" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_uri\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.lb_try_duration\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists +} + +@test "(caddy) invalid stored healthcheck path fails restart" { + run /bin/bash -c "dokku proxy:set $TEST_APP caddy" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "jq '.healthchecks.web[0].path = \"/health check\"' /var/lib/dokku/data/app-json/$TEST_APP/app.json > /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp && mv /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp /var/lib/dokku/data/app-json/$TEST_APP/app.json" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:restart $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_failure + assert_output_contains "Invalid app.json healthcheck path for caddy" -1 +} + +setup_caddy_readiness_healthcheck() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + convert_to_dockerfile "$APP" "$APP_REPO_DIR" + mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json" +} diff --git a/tests/unit/openresty.bats b/tests/unit/openresty.bats index 6751438ff..eee7e9e08 100644 --- a/tests/unit/openresty.bats +++ b/tests/unit/openresty.bats @@ -760,6 +760,175 @@ teardown() { assert_output_not_exists } +@test "(openresty) healthcheck labels from app.json" { + run /bin/bash -c "dokku proxy:set $TEST_APP openresty" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_openresty_readiness_healthcheck + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.upstream-max-fails\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "1" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.upstream-fail-timeout\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "2s" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.proxy-next-upstream-timeout\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "5s" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-path\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "/" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-interval\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "2" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-timeout\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "5" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-fall\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "3" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-host\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$TEST_APP.dokku.me" + + run /bin/bash -c "docker exec openresty-openresty-1 /usr/local/openresty/nginx/sbin/nginx -t" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker exec openresty-openresty-1 cat /etc/nginx/sites-enabled/sites.conf" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "max_fails=1 fail_timeout=2s;" + assert_output_contains "proxy_next_upstream_timeout 5s;" + + sleep 5 + + run /bin/bash -c "docker exec openresty-openresty-1 wget -qO- http://127.0.0.1:8999/upstream-healthcheck-status" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Upstream $TEST_APP-web-5000" + assert_output_contains " UP" + assert_output_contains "(NO checkers)" 0 + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + +@test "(openresty) no healthcheck labels without readiness check" { + run /bin/bash -c "dokku proxy:set $TEST_APP openresty" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.upstream-max-fails\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.proxy-next-upstream-timeout\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-path\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists +} + +@test "(openresty) healthcheck headers from app.json" { + run /bin/bash -c "dokku proxy:set $TEST_APP openresty" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_openresty_readiness_healthcheck_headers + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-header.X-Check\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "dokku" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-host\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "check.dokku.me" + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"openresty.healthcheck-header.Host\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_exists + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + +@test "(openresty) invalid stored healthcheck path fails restart" { + run /bin/bash -c "dokku proxy:set $TEST_APP openresty" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_openresty_readiness_healthcheck + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "jq '.healthchecks.web[0].path = \"/health check\"' /var/lib/dokku/data/app-json/$TEST_APP/app.json > /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp && mv /var/lib/dokku/data/app-json/$TEST_APP/app.json.tmp /var/lib/dokku/data/app-json/$TEST_APP/app.json" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:restart $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_failure + assert_output_contains "Invalid app.json healthcheck path for openresty" -1 +} + add_openresty_include() { local APP="$1" local APP_REPO_DIR="$2" @@ -786,3 +955,23 @@ add_openresty_include_unsafe() { mkdir -p "$APP_REPO_DIR/openresty/http-location-includes" printf '# location\n' >"$APP_REPO_DIR/openresty/http-location-includes/unsafe filename.conf" } + +setup_openresty_readiness_healthcheck() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + convert_to_dockerfile "$APP" "$APP_REPO_DIR" + mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json" +} + +setup_openresty_readiness_healthcheck_headers() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + convert_to_dockerfile "$APP" "$APP_REPO_DIR" + jq '.healthchecks.web[0].httpHeaders = [{"name": "X-Check", "value": "dokku"}, {"name": "Host", "value": "check.dokku.me"}]' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json" +} diff --git a/tests/unit/traefik.bats b/tests/unit/traefik.bats index a37ddc91e..6a3d1ac2f 100644 --- a/tests/unit/traefik.bats +++ b/tests/unit/traefik.bats @@ -943,7 +943,7 @@ setup_traefik_healthcheck() { [[ -z "$APP" ]] && local APP="$TEST_APP" APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" - mv "$APP_REPO_DIR/app-traefik.json" "$APP_REPO_DIR/app.json" + mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json" } setup_traefik_healthcheck_whitespace_path() { @@ -952,7 +952,7 @@ setup_traefik_healthcheck_whitespace_path() { [[ -z "$APP" ]] && local APP="$TEST_APP" APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" - jq '.healthchecks.web[0].path = "/health check"' "$APP_REPO_DIR/app-traefik.json" >"$APP_REPO_DIR/app.json" + jq '.healthchecks.web[0].path = "/health check"' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json" } setup_traefik_healthcheck_invalid_scheme() { @@ -961,5 +961,5 @@ setup_traefik_healthcheck_invalid_scheme() { [[ -z "$APP" ]] && local APP="$TEST_APP" APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" - jq '.healthchecks.web[0].scheme = "tcp"' "$APP_REPO_DIR/app-traefik.json" >"$APP_REPO_DIR/app.json" + jq '.healthchecks.web[0].scheme = "tcp"' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json" } From 06f80b1a93ded3d39f12baa8bb8cc0d10b6802bc Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 26 Sep 2026 01:17:30 -0400 Subject: [PATCH 2/4] fix: map caddy healthcheck headers and apply tls-internal Caddy healthchecks now honor the `attempts` and `httpHeaders` properties of the `app.json` readiness healthcheck, matching openresty. The `tls-internal` property was also never applied to app containers, and now takes effect for apps served over https. --- docs/networking/proxies/caddy.md | 8 ++- .../caddy-vhosts/docker-args-process-deploy | 47 ++++++++---- .../docker-args-process-deploy | 31 ++------ plugins/proxy/functions | 39 ++++++++++ tests/unit/caddy-vhosts-docker-args.bats | 72 +++++++++++++++++++ tests/unit/caddy.bats | 51 +++++++++++++ 6 files changed, 205 insertions(+), 43 deletions(-) diff --git a/docs/networking/proxies/caddy.md b/docs/networking/proxies/caddy.md index 3484e8e57..0a5f2243a 100644 --- a/docs/networking/proxies/caddy.md +++ b/docs/networking/proxies/caddy.md @@ -46,12 +46,14 @@ The following `app.json` healthcheck properties are mapped to Caddy labels: | `port` | `caddy.reverse_proxy.health_port` | The port to check | | `timeout` | `caddy.reverse_proxy.health_timeout` | Timeout in seconds (formatted as `Xs`, defaults to `5s`) | | `wait` | `caddy.reverse_proxy.health_interval` | Interval between checks in seconds (formatted as `Xs`, defaults to `5s`) | +| `attempts` | `caddy.reverse_proxy.health_fails` | Number of successive failed checks before a container is taken out of rotation (defaults to `3`) | +| `httpHeaders` | `caddy.reverse_proxy.health_headers.` | Headers to send with the check | Additionally, the following labels are generated: | Caddy Label | Value | Description | |-------------|-------|-------------| -| `caddy.reverse_proxy.health_headers.Host` | the first app domain | The `Host` header sent with the healthcheck, matching the header Dokku sends during deploy checks | +| `caddy.reverse_proxy.health_headers.Host` | the first app domain | The `Host` header sent with the healthcheck, matching the header Dokku sends during deploy checks. A `Host` header in `httpHeaders` takes precedence | | `caddy.reverse_proxy.lb_try_duration` | `5s` | How long a request that fails to connect is retried against other containers | | `caddy.reverse_proxy.lb_try_interval` | `250ms` | How long to wait between retries | @@ -75,7 +77,7 @@ Example `app.json` configuration: Caddy considers a new container healthy until its first check fails. The first check runs as soon as Caddy picks up the new container, and requests that reach the container before then and fail to connect are retried against the still-running old container. -The `scheme` property is not mapped, as Caddy uses the same transport for healthchecks as for proxied requests. Any of the generated labels may be overridden via the `caddy:labels:add` command. +Headers with values containing quotes, backslashes, or control characters are skipped, as are headers whose names contain a `.` or end in an `_` suffix, as these cannot be represented as Caddy labels. The `scheme` property is not mapped, as Caddy uses the same transport for healthchecks as for proxied requests. Any of the generated labels may be overridden via the `caddy:labels:add` command. > [!NOTE] > Only the first readiness healthcheck with a `path` property is used. Apps without one are routed as soon as the container is running. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#healthchecks-and-label-based-proxies) for more information. @@ -243,6 +245,8 @@ To switch to Caddy's internal TLS server for certificate provisioning, set the ` dokku caddy:set node-js-app tls-internal true ``` +As Caddy only serves https when the `letsencrypt-email` property is set, the internal TLS server is only used for apps when that property is also set. Apps will need to be rebuilt after changing this property. + The default value may also be configured globally with the `--global` flag. Per-app values take precedence over the global value when set. ```shell diff --git a/plugins/caddy-vhosts/docker-args-process-deploy b/plugins/caddy-vhosts/docker-args-process-deploy index 5235c2b67..e9074527c 100755 --- a/plugins/caddy-vhosts/docker-args-process-deploy +++ b/plugins/caddy-vhosts/docker-args-process-deploy @@ -95,7 +95,6 @@ trigger-caddy-vhosts-docker-args-process-deploy() { ssl_warning_mapping="https:443" if [[ "$tls_internal" == "true" ]]; then - output="--label caddy.tls=internal" if [[ "$has_443_mapping" == "false" ]]; then ssl_warning_mapping="http:80" proxy_host_https_port_candidate="$proxy_host_http_port_candidate" @@ -107,6 +106,9 @@ trigger-caddy-vhosts-docker-args-process-deploy() { scheme="http" if [[ -n "$letsencrypt_email" ]] && [[ "$has_443_mapping" == "true" ]]; then output="--label 'caddy=${caddy_domains}'" + if [[ "$tls_internal" == "true" ]]; then + output="$output --label caddy.tls=internal" + fi scheme="https" if [[ -z "$proxy_container_https_port" ]]; then warning_scheme="$(awk -F ':' '{ print $1 }' <<<"$ssl_warning_mapping")" @@ -129,7 +131,7 @@ trigger-caddy-vhosts-docker-args-process-deploy() { fi if [[ "$output" == *"caddy.reverse_proxy="* ]]; then - output="$output$(fn-caddy-healthcheck-labels "$APP" "$app_domains")" + output="$output$(fn-caddy-healthcheck-labels "$APP")" fi fi @@ -146,9 +148,8 @@ trigger-caddy-vhosts-docker-args-process-deploy() { fn-caddy-healthcheck-labels() { declare desc="outputs caddy reverse_proxy health check and retry labels derived from the app's web readiness healthcheck" - declare APP="$1" APP_DOMAINS="$2" - local healthcheck hc_path hc_port hc_timeout hc_interval hc_host domain - local -a domains + declare APP="$1" + local healthcheck hc_path hc_port hc_timeout hc_interval hc_attempts hc_host name value healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")" if [[ -z "$healthcheck" ]]; then @@ -159,28 +160,44 @@ fn-caddy-healthcheck-labels() { hc_port="$(echo "$healthcheck" | jq -r ".port // empty")" hc_timeout="$(echo "$healthcheck" | jq -r ".timeout // 5")" hc_interval="$(echo "$healthcheck" | jq -r ".wait // 5")" + hc_attempts="$(echo "$healthcheck" | jq -r ".attempts // 3")" if ! fn-proxy-healthcheck-path-is-valid "$hc_path"; then dokku_log_fail "Invalid app.json healthcheck path for caddy: path must start with a / and must not contain whitespace, control characters, quotes, or backslashes" fi - read -r -d '' -a domains <<<"$APP_DOMAINS" || true - for domain in "${domains[@]}"; do - if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then - continue - fi - - hc_host="$domain" - break - done + hc_host="$(fn-proxy-get-healthcheck-host "$APP" "$healthcheck")" echo -n " --label \"caddy.reverse_proxy.health_uri=$hc_path\"" [[ -n "$hc_port" ]] && echo -n " --label caddy.reverse_proxy.health_port=$hc_port" echo -n " --label caddy.reverse_proxy.health_interval=${hc_interval}s" echo -n " --label caddy.reverse_proxy.health_timeout=${hc_timeout}s" - [[ -n "$hc_host" ]] && echo -n " --label caddy.reverse_proxy.health_headers.Host=$hc_host" + echo -n " --label caddy.reverse_proxy.health_fails=$hc_attempts" + if [[ -n "$hc_host" ]] && fn-proxy-healthcheck-header-is-valid "Host" "$hc_host" && [[ "$hc_host" != *[[:space:]]* ]]; then + echo -n " --label caddy.reverse_proxy.health_headers.Host=$hc_host" + fi + + while IFS=$'\t' read -r name value; do + [[ -z "$name" ]] && continue + if ! fn-caddy-healthcheck-header-name-is-valid "$name" || ! fn-proxy-healthcheck-header-is-valid "$name" "$value"; then + dokku_log_warn "Skipping invalid app.json healthcheck header for caddy: $name" + continue + fi + + echo -n " --label 'caddy.reverse_proxy.health_headers.$name=\"$value\"'" + done < <(fn-proxy-get-healthcheck-headers "$healthcheck") + echo -n " --label caddy.reverse_proxy.lb_try_duration=5s" echo -n " --label caddy.reverse_proxy.lb_try_interval=250ms" } +fn-caddy-healthcheck-header-name-is-valid() { + declare desc="checks that a healthcheck header name can be used as a caddy label segment, which may not contain dots or end in an _ ordering suffix" + declare NAME="$1" + + [[ "$NAME" == *.* ]] && return 1 + [[ "$NAME" =~ _[0-9]+$ ]] && return 1 + return 0 +} + trigger-caddy-vhosts-docker-args-process-deploy "$@" diff --git a/plugins/openresty-vhosts/docker-args-process-deploy b/plugins/openresty-vhosts/docker-args-process-deploy index 9bbe1fb16..2b34911d8 100755 --- a/plugins/openresty-vhosts/docker-args-process-deploy +++ b/plugins/openresty-vhosts/docker-args-process-deploy @@ -222,8 +222,7 @@ trigger-openresty-vhosts-docker-args-process-deploy() { fn-openresty-healthcheck-labels() { declare desc="outputs openresty upstream healthcheck and failover labels derived from the app's web readiness healthcheck" declare APP="$1" - local healthcheck hc_path hc_port hc_scheme hc_timeout hc_interval hc_attempts hc_host domain name value - local -a domains + local healthcheck hc_path hc_port hc_scheme hc_timeout hc_interval hc_attempts hc_host name value healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")" if [[ -z "$healthcheck" ]]; then @@ -245,18 +244,7 @@ fn-openresty-healthcheck-labels() { dokku_log_fail "Invalid app.json healthcheck scheme for openresty: scheme must be one of http or https" fi - hc_host="$(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) == "host")) | .[0].value // empty')" - if [[ -z "$hc_host" ]]; then - read -r -d '' -a domains <<<"$(plugn trigger domains-list "$APP")" || true - for domain in "${domains[@]}"; do - if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then - continue - fi - - hc_host="$domain" - break - done - fi + hc_host="$(fn-proxy-get-healthcheck-host "$APP" "$healthcheck")" echo -n " '--label=openresty.upstream-max-fails=1'" echo -n " '--label=openresty.upstream-fail-timeout=${hc_interval}s'" @@ -267,28 +255,19 @@ fn-openresty-healthcheck-labels() { echo -n " '--label=openresty.healthcheck-timeout=$hc_timeout'" echo -n " '--label=openresty.healthcheck-interval=$hc_interval'" echo -n " '--label=openresty.healthcheck-fall=$hc_attempts'" - if [[ -n "$hc_host" ]] && fn-openresty-healthcheck-header-is-valid "Host" "$hc_host"; then + if [[ -n "$hc_host" ]] && fn-proxy-healthcheck-header-is-valid "Host" "$hc_host"; then echo -n " '--label=openresty.healthcheck-host=$hc_host'" fi while IFS=$'\t' read -r name value; do [[ -z "$name" ]] && continue - if ! fn-openresty-healthcheck-header-is-valid "$name" "$value"; then + if ! fn-proxy-healthcheck-header-is-valid "$name" "$value"; then dokku_log_warn "Skipping invalid app.json healthcheck header for openresty: $name" continue fi echo -n " '--label=openresty.healthcheck-header.$name=$value'" - done < <(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) != "host")) | .[] | [.name, .value] | @tsv') -} - -fn-openresty-healthcheck-header-is-valid() { - declare desc="checks that a healthcheck header name is a token and its value is free of control, quote, and backslash characters" - declare NAME="$1" VALUE="$2" - - [[ "$NAME" =~ ^[A-Za-z0-9!#$%\&*+.^_\`\|~-]+$ ]] || return 1 - [[ "$VALUE" =~ [[:cntrl:]\"\'\\] ]] && return 1 - return 0 + done < <(fn-proxy-get-healthcheck-headers "$healthcheck") } trigger-openresty-vhosts-docker-args-process-deploy "$@" diff --git a/plugins/proxy/functions b/plugins/proxy/functions index 46e994db3..b235a0d2a 100755 --- a/plugins/proxy/functions +++ b/plugins/proxy/functions @@ -35,6 +35,45 @@ fn-proxy-healthcheck-path-is-valid() { return 0 } +fn-proxy-healthcheck-header-is-valid() { + declare desc="checks that an app.json healthcheck header name is a token and its value is free of control, quote, and backslash characters" + declare NAME="$1" VALUE="$2" + + [[ "$NAME" =~ ^[A-Za-z0-9!#$%\&*+.^_\`\|~-]+$ ]] || return 1 + [[ "$VALUE" =~ [[:cntrl:]\"\'\\] ]] && return 1 + return 0 +} + +fn-proxy-get-healthcheck-host() { + declare desc="outputs the host header for a healthcheck, preferring an app.json Host header over the first app domain" + declare APP="$1" HEALTHCHECK="$2" + local domain hc_host + local -a domains + + hc_host="$(echo "$HEALTHCHECK" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) == "host")) | .[0].value // empty')" + if [[ -n "$hc_host" ]]; then + echo "$hc_host" + return + fi + + read -r -d '' -a domains <<<"$(plugn trigger domains-list "$APP")" || true + for domain in "${domains[@]}"; do + if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then + continue + fi + + echo "$domain" + return + done +} + +fn-proxy-get-healthcheck-headers() { + declare desc="outputs the tab-separated name and value of each non-Host app.json healthcheck header" + declare HEALTHCHECK="$1" + + echo "$HEALTHCHECK" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) != "host")) | .[] | [.name, .value] | @tsv' +} + fn-proxy-healthcheck-scheme-is-valid() { declare desc="checks that an app.json healthcheck scheme is either http or https" declare HC_SCHEME="$1" diff --git a/tests/unit/caddy-vhosts-docker-args.bats b/tests/unit/caddy-vhosts-docker-args.bats index 223b648ce..5b4ec9c18 100644 --- a/tests/unit/caddy-vhosts-docker-args.bats +++ b/tests/unit/caddy-vhosts-docker-args.bats @@ -237,3 +237,75 @@ invoke_caddy_docker_args() { assert_failure assert_output_contains "Invalid app.json healthcheck path for caddy" } + +@test "(caddy-vhosts) readiness healthcheck maps attempts and headers" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","attempts":4,"httpHeaders":[{"name":"X-Check","value":"dokku check"},{"name":"host","value":"check.example.com"}]}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label caddy.reverse_proxy.health_fails=4" + assert_output_contains "--label 'caddy.reverse_proxy.health_headers.X-Check=\"dokku check\"'" + assert_output_contains "--label caddy.reverse_proxy.health_headers.Host=check.example.com" + assert_output_contains "health_headers.Host=example.com" 0 + assert_output_contains "health_headers.host" 0 +} + +@test "(caddy-vhosts) readiness healthcheck defaults attempts to 3" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/"}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label caddy.reverse_proxy.health_fails=3" +} + +@test "(caddy-vhosts) invalid healthcheck headers are skipped" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","httpHeaders":[{"name":"X.Dotted","value":"a"},{"name":"X-Order_1","value":"a"},{"name":"X-Quote","value":"it'"'"'s"},{"name":"Bad Name","value":"a"},{"name":"X-Good","value":"ok"}]}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Skipping invalid app.json healthcheck header for caddy" 4 + assert_output_contains "health_headers.X-Good=" 1 + assert_output_contains "health_headers.X.Dotted" 0 + assert_output_contains "health_headers.X-Order_1" 0 + assert_output_contains "health_headers.X-Quote" 0 +} + +@test "(caddy-vhosts) tls-internal emits caddy.tls=internal alongside https site labels" { + STUB_DOMAINS="example.com" + STUB_PORTS="$(printf 'http:80:5000\nhttps:443:5000')" + STUB_LETSENCRYPT_EMAIL="admin@example.com" + STUB_TLS_INTERNAL="true" + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label 'caddy=example.com'" + assert_output_contains "--label caddy.tls=internal" + assert_output_contains "caddy.reverse_proxy=" +} + +@test "(caddy-vhosts) tls-internal does not emit caddy.tls=internal for http-only sites" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_TLS_INTERNAL="true" + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label 'caddy=example.com:80'" + assert_output_contains "caddy.tls=internal" 0 +} diff --git a/tests/unit/caddy.bats b/tests/unit/caddy.bats index 5d8342bb6..4f7ebe7c9 100644 --- a/tests/unit/caddy.bats +++ b/tests/unit/caddy.bats @@ -582,6 +582,47 @@ teardown() { assert_output_contains "Invalid app.json healthcheck path for caddy" -1 } +@test "(caddy) healthcheck headers from app.json" { + run /bin/bash -c "dokku proxy:set $TEST_APP caddy" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck_headers + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_headers.X-Check\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '"dokku check"' + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_fails\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "2" + + sleep 5 + + run /bin/bash -c "docker exec caddy-caddy-1 cat /config/caddy/Caddyfile.autosave" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'X-Check "dokku check"' + assert_output_contains "health_fails 2" + + run /bin/bash -c "docker logs caddy-caddy-1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Failed to convert caddyfile" 0 + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + setup_caddy_readiness_healthcheck() { local APP="$1" local APP_REPO_DIR="$2" @@ -591,3 +632,13 @@ setup_caddy_readiness_healthcheck() { convert_to_dockerfile "$APP" "$APP_REPO_DIR" mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json" } + +setup_caddy_readiness_healthcheck_headers() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + convert_to_dockerfile "$APP" "$APP_REPO_DIR" + jq '.healthchecks.web[0].attempts = 2 | .healthchecks.web[0].httpHeaders = [{"name": "X-Check", "value": "dokku check"}]' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json" +} From fba0681e5b4900e7b4269e5fb7764d75a3fa14b6 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 26 Sep 2026 01:17:30 -0400 Subject: [PATCH 3/4] fix: avoid glob expansion of app domains A catch-all `*` domain or a wildcard ssl hostname could be expanded into file names from the current directory when generating app urls, healthcheck `Host` headers, or the nginx ssl server names. --- plugins/domains/internal-functions | 8 ++++--- plugins/nginx-vhosts/functions | 4 +++- plugins/scheduler-docker-local/check-deploy | 8 ++++--- tests/unit/domains.bats | 25 +++++++++++++++++++++ 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/plugins/domains/internal-functions b/plugins/domains/internal-functions index 04f68bef6..dfcfbb77a 100644 --- a/plugins/domains/internal-functions +++ b/plugins/domains/internal-functions @@ -38,9 +38,11 @@ fn-domains-global-vhosts() { fn-domains-generate-urls() { declare APP="$1" DEFAULT_SCHEME="$2" DEFAULT_LISTEN_PORT="$3" - local app_vhosts="$(plugn trigger domains-list "$APP")" - if [[ -n "$app_vhosts" ]]; then - for app_vhost in $app_vhosts; do + local app_vhost + local -a app_vhosts + read -r -d '' -a app_vhosts <<<"$(plugn trigger domains-list "$APP")" || true + if [[ "${#app_vhosts[@]}" -gt 0 ]]; then + for app_vhost in "${app_vhosts[@]}"; do fn-domains-generate-urls-from-config "$APP" "$DEFAULT_SCHEME" "$app_vhost" "$DEFAULT_LISTEN_PORT" done else diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index 970f08062..d2a11212b 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -252,7 +252,9 @@ fn-nginx-vhosts-render-template() { SSL_VHOSTS=$(<"$DOKKU_ROOT/VHOST") fi local host - for host in $SSL_VHOSTS; do + local -a ssl_vhosts + read -r -d '' -a ssl_vhosts <<<"$SSL_VHOSTS" || true + for host in "${ssl_vhosts[@]}"; do if [[ ! $NOSSL_SERVER_NAME =~ (^|[[:space:]])$host($|[[:space:]]) ]]; then SSL_SERVER_NAME="${host}${SSL_SERVER_NAME:+ $SSL_SERVER_NAME}" fi diff --git a/plugins/scheduler-docker-local/check-deploy b/plugins/scheduler-docker-local/check-deploy index 8772aefd6..b9bdb1645 100755 --- a/plugins/scheduler-docker-local/check-deploy +++ b/plugins/scheduler-docker-local/check-deploy @@ -145,9 +145,11 @@ trigger-scheduler-docker-local-check-deploy() { ARG_ARRAY+=("$DOKKU_APP_LISTEN_PORT") fi - local app_vhosts="$(plugn trigger domains-list "$APP")" - if [[ -n "$app_vhosts" ]]; then - for app_vhost in $app_vhosts; do + local app_vhost + local -a app_vhosts + read -r -d '' -a app_vhosts <<<"$(plugn trigger domains-list "$APP")" || true + if [[ "${#app_vhosts[@]}" -gt 0 ]]; then + for app_vhost in "${app_vhosts[@]}"; do if [[ "$app_vhost" == "localhost" ]] || [[ "$app_vhost" == '*' ]]; then continue fi diff --git a/tests/unit/domains.bats b/tests/unit/domains.bats index bbf965fb2..6edd71f64 100644 --- a/tests/unit/domains.bats +++ b/tests/unit/domains.bats @@ -533,3 +533,28 @@ teardown() { assert_success assert_output "http:2019:2019 http:80:80 https:443:443 udp:443:443" } + +@test "(domains) urls do not glob expand a catch-all domain" { + local tmp_dir + tmp_dir="$(mktemp -d)" + touch "$tmp_dir/glob-canary" + chmod 755 "$tmp_dir" + + run /bin/bash -c "dokku proxy:set $TEST_APP caddy" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP '*'" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "cd $tmp_dir && dokku urls $TEST_APP" + echo "output: $output" + echo "status: $status" + rm -rf "$tmp_dir" + assert_success + assert_output_contains "glob-canary" 0 + assert_output_contains "http://*" +} From 1086de5aae72a7913737fb168eaa0d0b97e38cc9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 26 Sep 2026 01:22:19 -0400 Subject: [PATCH 4/4] test: read caddy config without exec in the container --- tests/unit/caddy.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/caddy.bats b/tests/unit/caddy.bats index 4f7ebe7c9..b8be0c5c0 100644 --- a/tests/unit/caddy.bats +++ b/tests/unit/caddy.bats @@ -607,7 +607,7 @@ teardown() { sleep 5 - run /bin/bash -c "docker exec caddy-caddy-1 cat /config/caddy/Caddyfile.autosave" + run /bin/bash -c "docker cp caddy-caddy-1:/config/caddy/Caddyfile.autosave - | tar -xO" echo "output: $output" echo "status: $status" assert_success