diff --git a/docs/networking/proxies/caddy.md b/docs/networking/proxies/caddy.md index 3484e8e57..0a5f2243a 100644 --- a/docs/networking/proxies/caddy.md +++ b/docs/networking/proxies/caddy.md @@ -46,12 +46,14 @@ The following `app.json` healthcheck properties are mapped to Caddy labels: | `port` | `caddy.reverse_proxy.health_port` | The port to check | | `timeout` | `caddy.reverse_proxy.health_timeout` | Timeout in seconds (formatted as `Xs`, defaults to `5s`) | | `wait` | `caddy.reverse_proxy.health_interval` | Interval between checks in seconds (formatted as `Xs`, defaults to `5s`) | +| `attempts` | `caddy.reverse_proxy.health_fails` | Number of successive failed checks before a container is taken out of rotation (defaults to `3`) | +| `httpHeaders` | `caddy.reverse_proxy.health_headers.` | Headers to send with the check | Additionally, the following labels are generated: | Caddy Label | Value | Description | |-------------|-------|-------------| -| `caddy.reverse_proxy.health_headers.Host` | the first app domain | The `Host` header sent with the healthcheck, matching the header Dokku sends during deploy checks | +| `caddy.reverse_proxy.health_headers.Host` | the first app domain | The `Host` header sent with the healthcheck, matching the header Dokku sends during deploy checks. A `Host` header in `httpHeaders` takes precedence | | `caddy.reverse_proxy.lb_try_duration` | `5s` | How long a request that fails to connect is retried against other containers | | `caddy.reverse_proxy.lb_try_interval` | `250ms` | How long to wait between retries | @@ -75,7 +77,7 @@ Example `app.json` configuration: Caddy considers a new container healthy until its first check fails. The first check runs as soon as Caddy picks up the new container, and requests that reach the container before then and fail to connect are retried against the still-running old container. -The `scheme` property is not mapped, as Caddy uses the same transport for healthchecks as for proxied requests. Any of the generated labels may be overridden via the `caddy:labels:add` command. +Headers with values containing quotes, backslashes, or control characters are skipped, as are headers whose names contain a `.` or end in an `_` suffix, as these cannot be represented as Caddy labels. The `scheme` property is not mapped, as Caddy uses the same transport for healthchecks as for proxied requests. Any of the generated labels may be overridden via the `caddy:labels:add` command. > [!NOTE] > Only the first readiness healthcheck with a `path` property is used. Apps without one are routed as soon as the container is running. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#healthchecks-and-label-based-proxies) for more information. @@ -243,6 +245,8 @@ To switch to Caddy's internal TLS server for certificate provisioning, set the ` dokku caddy:set node-js-app tls-internal true ``` +As Caddy only serves https when the `letsencrypt-email` property is set, the internal TLS server is only used for apps when that property is also set. Apps will need to be rebuilt after changing this property. + The default value may also be configured globally with the `--global` flag. Per-app values take precedence over the global value when set. ```shell diff --git a/plugins/caddy-vhosts/docker-args-process-deploy b/plugins/caddy-vhosts/docker-args-process-deploy index 5235c2b67..e9074527c 100755 --- a/plugins/caddy-vhosts/docker-args-process-deploy +++ b/plugins/caddy-vhosts/docker-args-process-deploy @@ -95,7 +95,6 @@ trigger-caddy-vhosts-docker-args-process-deploy() { ssl_warning_mapping="https:443" if [[ "$tls_internal" == "true" ]]; then - output="--label caddy.tls=internal" if [[ "$has_443_mapping" == "false" ]]; then ssl_warning_mapping="http:80" proxy_host_https_port_candidate="$proxy_host_http_port_candidate" @@ -107,6 +106,9 @@ trigger-caddy-vhosts-docker-args-process-deploy() { scheme="http" if [[ -n "$letsencrypt_email" ]] && [[ "$has_443_mapping" == "true" ]]; then output="--label 'caddy=${caddy_domains}'" + if [[ "$tls_internal" == "true" ]]; then + output="$output --label caddy.tls=internal" + fi scheme="https" if [[ -z "$proxy_container_https_port" ]]; then warning_scheme="$(awk -F ':' '{ print $1 }' <<<"$ssl_warning_mapping")" @@ -129,7 +131,7 @@ trigger-caddy-vhosts-docker-args-process-deploy() { fi if [[ "$output" == *"caddy.reverse_proxy="* ]]; then - output="$output$(fn-caddy-healthcheck-labels "$APP" "$app_domains")" + output="$output$(fn-caddy-healthcheck-labels "$APP")" fi fi @@ -146,9 +148,8 @@ trigger-caddy-vhosts-docker-args-process-deploy() { fn-caddy-healthcheck-labels() { declare desc="outputs caddy reverse_proxy health check and retry labels derived from the app's web readiness healthcheck" - declare APP="$1" APP_DOMAINS="$2" - local healthcheck hc_path hc_port hc_timeout hc_interval hc_host domain - local -a domains + declare APP="$1" + local healthcheck hc_path hc_port hc_timeout hc_interval hc_attempts hc_host name value healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")" if [[ -z "$healthcheck" ]]; then @@ -159,28 +160,44 @@ fn-caddy-healthcheck-labels() { hc_port="$(echo "$healthcheck" | jq -r ".port // empty")" hc_timeout="$(echo "$healthcheck" | jq -r ".timeout // 5")" hc_interval="$(echo "$healthcheck" | jq -r ".wait // 5")" + hc_attempts="$(echo "$healthcheck" | jq -r ".attempts // 3")" if ! fn-proxy-healthcheck-path-is-valid "$hc_path"; then dokku_log_fail "Invalid app.json healthcheck path for caddy: path must start with a / and must not contain whitespace, control characters, quotes, or backslashes" fi - read -r -d '' -a domains <<<"$APP_DOMAINS" || true - for domain in "${domains[@]}"; do - if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then - continue - fi - - hc_host="$domain" - break - done + hc_host="$(fn-proxy-get-healthcheck-host "$APP" "$healthcheck")" echo -n " --label \"caddy.reverse_proxy.health_uri=$hc_path\"" [[ -n "$hc_port" ]] && echo -n " --label caddy.reverse_proxy.health_port=$hc_port" echo -n " --label caddy.reverse_proxy.health_interval=${hc_interval}s" echo -n " --label caddy.reverse_proxy.health_timeout=${hc_timeout}s" - [[ -n "$hc_host" ]] && echo -n " --label caddy.reverse_proxy.health_headers.Host=$hc_host" + echo -n " --label caddy.reverse_proxy.health_fails=$hc_attempts" + if [[ -n "$hc_host" ]] && fn-proxy-healthcheck-header-is-valid "Host" "$hc_host" && [[ "$hc_host" != *[[:space:]]* ]]; then + echo -n " --label caddy.reverse_proxy.health_headers.Host=$hc_host" + fi + + while IFS=$'\t' read -r name value; do + [[ -z "$name" ]] && continue + if ! fn-caddy-healthcheck-header-name-is-valid "$name" || ! fn-proxy-healthcheck-header-is-valid "$name" "$value"; then + dokku_log_warn "Skipping invalid app.json healthcheck header for caddy: $name" + continue + fi + + echo -n " --label 'caddy.reverse_proxy.health_headers.$name=\"$value\"'" + done < <(fn-proxy-get-healthcheck-headers "$healthcheck") + echo -n " --label caddy.reverse_proxy.lb_try_duration=5s" echo -n " --label caddy.reverse_proxy.lb_try_interval=250ms" } +fn-caddy-healthcheck-header-name-is-valid() { + declare desc="checks that a healthcheck header name can be used as a caddy label segment, which may not contain dots or end in an _ ordering suffix" + declare NAME="$1" + + [[ "$NAME" == *.* ]] && return 1 + [[ "$NAME" =~ _[0-9]+$ ]] && return 1 + return 0 +} + trigger-caddy-vhosts-docker-args-process-deploy "$@" diff --git a/plugins/openresty-vhosts/docker-args-process-deploy b/plugins/openresty-vhosts/docker-args-process-deploy index 9bbe1fb16..2b34911d8 100755 --- a/plugins/openresty-vhosts/docker-args-process-deploy +++ b/plugins/openresty-vhosts/docker-args-process-deploy @@ -222,8 +222,7 @@ trigger-openresty-vhosts-docker-args-process-deploy() { fn-openresty-healthcheck-labels() { declare desc="outputs openresty upstream healthcheck and failover labels derived from the app's web readiness healthcheck" declare APP="$1" - local healthcheck hc_path hc_port hc_scheme hc_timeout hc_interval hc_attempts hc_host domain name value - local -a domains + local healthcheck hc_path hc_port hc_scheme hc_timeout hc_interval hc_attempts hc_host name value healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")" if [[ -z "$healthcheck" ]]; then @@ -245,18 +244,7 @@ fn-openresty-healthcheck-labels() { dokku_log_fail "Invalid app.json healthcheck scheme for openresty: scheme must be one of http or https" fi - hc_host="$(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) == "host")) | .[0].value // empty')" - if [[ -z "$hc_host" ]]; then - read -r -d '' -a domains <<<"$(plugn trigger domains-list "$APP")" || true - for domain in "${domains[@]}"; do - if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then - continue - fi - - hc_host="$domain" - break - done - fi + hc_host="$(fn-proxy-get-healthcheck-host "$APP" "$healthcheck")" echo -n " '--label=openresty.upstream-max-fails=1'" echo -n " '--label=openresty.upstream-fail-timeout=${hc_interval}s'" @@ -267,28 +255,19 @@ fn-openresty-healthcheck-labels() { echo -n " '--label=openresty.healthcheck-timeout=$hc_timeout'" echo -n " '--label=openresty.healthcheck-interval=$hc_interval'" echo -n " '--label=openresty.healthcheck-fall=$hc_attempts'" - if [[ -n "$hc_host" ]] && fn-openresty-healthcheck-header-is-valid "Host" "$hc_host"; then + if [[ -n "$hc_host" ]] && fn-proxy-healthcheck-header-is-valid "Host" "$hc_host"; then echo -n " '--label=openresty.healthcheck-host=$hc_host'" fi while IFS=$'\t' read -r name value; do [[ -z "$name" ]] && continue - if ! fn-openresty-healthcheck-header-is-valid "$name" "$value"; then + if ! fn-proxy-healthcheck-header-is-valid "$name" "$value"; then dokku_log_warn "Skipping invalid app.json healthcheck header for openresty: $name" continue fi echo -n " '--label=openresty.healthcheck-header.$name=$value'" - done < <(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) != "host")) | .[] | [.name, .value] | @tsv') -} - -fn-openresty-healthcheck-header-is-valid() { - declare desc="checks that a healthcheck header name is a token and its value is free of control, quote, and backslash characters" - declare NAME="$1" VALUE="$2" - - [[ "$NAME" =~ ^[A-Za-z0-9!#$%\&*+.^_\`\|~-]+$ ]] || return 1 - [[ "$VALUE" =~ [[:cntrl:]\"\'\\] ]] && return 1 - return 0 + done < <(fn-proxy-get-healthcheck-headers "$healthcheck") } trigger-openresty-vhosts-docker-args-process-deploy "$@" diff --git a/plugins/proxy/functions b/plugins/proxy/functions index 46e994db3..b235a0d2a 100755 --- a/plugins/proxy/functions +++ b/plugins/proxy/functions @@ -35,6 +35,45 @@ fn-proxy-healthcheck-path-is-valid() { return 0 } +fn-proxy-healthcheck-header-is-valid() { + declare desc="checks that an app.json healthcheck header name is a token and its value is free of control, quote, and backslash characters" + declare NAME="$1" VALUE="$2" + + [[ "$NAME" =~ ^[A-Za-z0-9!#$%\&*+.^_\`\|~-]+$ ]] || return 1 + [[ "$VALUE" =~ [[:cntrl:]\"\'\\] ]] && return 1 + return 0 +} + +fn-proxy-get-healthcheck-host() { + declare desc="outputs the host header for a healthcheck, preferring an app.json Host header over the first app domain" + declare APP="$1" HEALTHCHECK="$2" + local domain hc_host + local -a domains + + hc_host="$(echo "$HEALTHCHECK" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) == "host")) | .[0].value // empty')" + if [[ -n "$hc_host" ]]; then + echo "$hc_host" + return + fi + + read -r -d '' -a domains <<<"$(plugn trigger domains-list "$APP")" || true + for domain in "${domains[@]}"; do + if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then + continue + fi + + echo "$domain" + return + done +} + +fn-proxy-get-healthcheck-headers() { + declare desc="outputs the tab-separated name and value of each non-Host app.json healthcheck header" + declare HEALTHCHECK="$1" + + echo "$HEALTHCHECK" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) != "host")) | .[] | [.name, .value] | @tsv' +} + fn-proxy-healthcheck-scheme-is-valid() { declare desc="checks that an app.json healthcheck scheme is either http or https" declare HC_SCHEME="$1" diff --git a/tests/unit/caddy-vhosts-docker-args.bats b/tests/unit/caddy-vhosts-docker-args.bats index 223b648ce..5b4ec9c18 100644 --- a/tests/unit/caddy-vhosts-docker-args.bats +++ b/tests/unit/caddy-vhosts-docker-args.bats @@ -237,3 +237,75 @@ invoke_caddy_docker_args() { assert_failure assert_output_contains "Invalid app.json healthcheck path for caddy" } + +@test "(caddy-vhosts) readiness healthcheck maps attempts and headers" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","attempts":4,"httpHeaders":[{"name":"X-Check","value":"dokku check"},{"name":"host","value":"check.example.com"}]}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label caddy.reverse_proxy.health_fails=4" + assert_output_contains "--label 'caddy.reverse_proxy.health_headers.X-Check=\"dokku check\"'" + assert_output_contains "--label caddy.reverse_proxy.health_headers.Host=check.example.com" + assert_output_contains "health_headers.Host=example.com" 0 + assert_output_contains "health_headers.host" 0 +} + +@test "(caddy-vhosts) readiness healthcheck defaults attempts to 3" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/"}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label caddy.reverse_proxy.health_fails=3" +} + +@test "(caddy-vhosts) invalid healthcheck headers are skipped" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","httpHeaders":[{"name":"X.Dotted","value":"a"},{"name":"X-Order_1","value":"a"},{"name":"X-Quote","value":"it'"'"'s"},{"name":"Bad Name","value":"a"},{"name":"X-Good","value":"ok"}]}]}}' + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Skipping invalid app.json healthcheck header for caddy" 4 + assert_output_contains "health_headers.X-Good=" 1 + assert_output_contains "health_headers.X.Dotted" 0 + assert_output_contains "health_headers.X-Order_1" 0 + assert_output_contains "health_headers.X-Quote" 0 +} + +@test "(caddy-vhosts) tls-internal emits caddy.tls=internal alongside https site labels" { + STUB_DOMAINS="example.com" + STUB_PORTS="$(printf 'http:80:5000\nhttps:443:5000')" + STUB_LETSENCRYPT_EMAIL="admin@example.com" + STUB_TLS_INTERNAL="true" + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label 'caddy=example.com'" + assert_output_contains "--label caddy.tls=internal" + assert_output_contains "caddy.reverse_proxy=" +} + +@test "(caddy-vhosts) tls-internal does not emit caddy.tls=internal for http-only sites" { + STUB_DOMAINS="example.com" + STUB_PORTS="http:80:5000" + STUB_TLS_INTERNAL="true" + + run invoke_caddy_docker_args + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "--label 'caddy=example.com:80'" + assert_output_contains "caddy.tls=internal" 0 +} diff --git a/tests/unit/caddy.bats b/tests/unit/caddy.bats index 5d8342bb6..4f7ebe7c9 100644 --- a/tests/unit/caddy.bats +++ b/tests/unit/caddy.bats @@ -582,6 +582,47 @@ teardown() { assert_output_contains "Invalid app.json healthcheck path for caddy" -1 } +@test "(caddy) healthcheck headers from app.json" { + run /bin/bash -c "dokku proxy:set $TEST_APP caddy" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck_headers + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_headers.X-Check\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '"dokku check"' + + run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_fails\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "2" + + sleep 5 + + run /bin/bash -c "docker exec caddy-caddy-1 cat /config/caddy/Caddyfile.autosave" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'X-Check "dokku check"' + assert_output_contains "health_fails 2" + + run /bin/bash -c "docker logs caddy-caddy-1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Failed to convert caddyfile" 0 + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + setup_caddy_readiness_healthcheck() { local APP="$1" local APP_REPO_DIR="$2" @@ -591,3 +632,13 @@ setup_caddy_readiness_healthcheck() { convert_to_dockerfile "$APP" "$APP_REPO_DIR" mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json" } + +setup_caddy_readiness_healthcheck_headers() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + convert_to_dockerfile "$APP" "$APP_REPO_DIR" + jq '.healthchecks.web[0].attempts = 2 | .healthchecks.web[0].httpHeaders = [{"name": "X-Check", "value": "dokku check"}]' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json" +}