fix: map caddy healthcheck headers and apply tls-internal

Caddy healthchecks now honor the `attempts` and `httpHeaders` properties of the `app.json` readiness healthcheck, matching openresty. The `tls-internal` property was also never applied to app containers, and now takes effect for apps served over https.

docs/networking/proxies/caddy.md

@@ -46,12 +46,14 @@ The following `app.json` healthcheck properties are mapped to Caddy labels:
 | `port`            | `caddy.reverse_proxy.health_port`     | The port to check |
 | `timeout`         | `caddy.reverse_proxy.health_timeout`  | Timeout in seconds (formatted as `Xs`, defaults to `5s`) |
 | `wait`            | `caddy.reverse_proxy.health_interval` | Interval between checks in seconds (formatted as `Xs`, defaults to `5s`) |
+| `attempts`        | `caddy.reverse_proxy.health_fails`    | Number of successive failed checks before a container is taken out of rotation (defaults to `3`) |
+| `httpHeaders`     | `caddy.reverse_proxy.health_headers.<name>` | Headers to send with the check |
 
 Additionally, the following labels are generated:
 
 | Caddy Label | Value | Description |
 |-------------|-------|-------------|
-| `caddy.reverse_proxy.health_headers.Host` | the first app domain | The `Host` header sent with the healthcheck, matching the header Dokku sends during deploy checks |
+| `caddy.reverse_proxy.health_headers.Host` | the first app domain | The `Host` header sent with the healthcheck, matching the header Dokku sends during deploy checks. A `Host` header in `httpHeaders` takes precedence |
 | `caddy.reverse_proxy.lb_try_duration` | `5s` | How long a request that fails to connect is retried against other containers |
 | `caddy.reverse_proxy.lb_try_interval` | `250ms` | How long to wait between retries |
 
@@ -75,7 +77,7 @@ Example `app.json` configuration:
 
 Caddy considers a new container healthy until its first check fails. The first check runs as soon as Caddy picks up the new container, and requests that reach the container before then and fail to connect are retried against the still-running old container.
 
-The `scheme` property is not mapped, as Caddy uses the same transport for healthchecks as for proxied requests. Any of the generated labels may be overridden via the `caddy:labels:add` command.
+Headers with values containing quotes, backslashes, or control characters are skipped, as are headers whose names contain a `.` or end in an `_<number>` suffix, as these cannot be represented as Caddy labels. The `scheme` property is not mapped, as Caddy uses the same transport for healthchecks as for proxied requests. Any of the generated labels may be overridden via the `caddy:labels:add` command.
 
 > [!NOTE]
 > Only the first readiness healthcheck with a `path` property is used. Apps without one are routed as soon as the container is running. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#healthchecks-and-label-based-proxies) for more information.
@@ -243,6 +245,8 @@ To switch to Caddy's internal TLS server for certificate provisioning, set the `
 dokku caddy:set node-js-app tls-internal true
 ```
 
+As Caddy only serves https when the `letsencrypt-email` property is set, the internal TLS server is only used for apps when that property is also set. Apps will need to be rebuilt after changing this property.
+
 The default value may also be configured globally with the `--global` flag. Per-app values take precedence over the global value when set.
 
 ```shell

plugins/caddy-vhosts/docker-args-process-deploy

@@ -95,7 +95,6 @@ trigger-caddy-vhosts-docker-args-process-deploy() {
 
     ssl_warning_mapping="https:443"
     if [[ "$tls_internal" == "true" ]]; then
-      output="--label caddy.tls=internal"
       if [[ "$has_443_mapping" == "false" ]]; then
         ssl_warning_mapping="http:80"
         proxy_host_https_port_candidate="$proxy_host_http_port_candidate"
@@ -107,6 +106,9 @@ trigger-caddy-vhosts-docker-args-process-deploy() {
     scheme="http"
     if [[ -n "$letsencrypt_email" ]] && [[ "$has_443_mapping" == "true" ]]; then
       output="--label 'caddy=${caddy_domains}'"
+      if [[ "$tls_internal" == "true" ]]; then
+        output="$output --label caddy.tls=internal"
+      fi
       scheme="https"
       if [[ -z "$proxy_container_https_port" ]]; then
         warning_scheme="$(awk -F ':' '{ print $1 }' <<<"$ssl_warning_mapping")"
@@ -129,7 +131,7 @@ trigger-caddy-vhosts-docker-args-process-deploy() {
     fi
 
     if [[ "$output" == *"caddy.reverse_proxy="* ]]; then
-      output="$output$(fn-caddy-healthcheck-labels "$APP" "$app_domains")"
+      output="$output$(fn-caddy-healthcheck-labels "$APP")"
     fi
   fi
 
@@ -146,9 +148,8 @@ trigger-caddy-vhosts-docker-args-process-deploy() {
 
 fn-caddy-healthcheck-labels() {
   declare desc="outputs caddy reverse_proxy health check and retry labels derived from the app's web readiness healthcheck"
-  declare APP="$1" APP_DOMAINS="$2"
+  declare APP="$1"
-  local healthcheck hc_path hc_port hc_timeout hc_interval hc_host domain
+  local healthcheck hc_path hc_port hc_timeout hc_interval hc_attempts hc_host name value
-  local -a domains
 
   healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")"
   if [[ -z "$healthcheck" ]]; then
@@ -159,28 +160,44 @@ fn-caddy-healthcheck-labels() {
   hc_port="$(echo "$healthcheck" | jq -r ".port // empty")"
   hc_timeout="$(echo "$healthcheck" | jq -r ".timeout // 5")"
   hc_interval="$(echo "$healthcheck" | jq -r ".wait // 5")"
+  hc_attempts="$(echo "$healthcheck" | jq -r ".attempts // 3")"
 
   if ! fn-proxy-healthcheck-path-is-valid "$hc_path"; then
     dokku_log_fail "Invalid app.json healthcheck path for caddy: path must start with a / and must not contain whitespace, control characters, quotes, or backslashes"
   fi
 
-  read -r -d '' -a domains <<<"$APP_DOMAINS" || true
+  hc_host="$(fn-proxy-get-healthcheck-host "$APP" "$healthcheck")"
-  for domain in "${domains[@]}"; do
-    if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then
-      continue
-    fi
-
-    hc_host="$domain"
-    break
-  done
 
   echo -n " --label \"caddy.reverse_proxy.health_uri=$hc_path\""
   [[ -n "$hc_port" ]] && echo -n " --label caddy.reverse_proxy.health_port=$hc_port"
   echo -n " --label caddy.reverse_proxy.health_interval=${hc_interval}s"
   echo -n " --label caddy.reverse_proxy.health_timeout=${hc_timeout}s"
-  [[ -n "$hc_host" ]] && echo -n " --label caddy.reverse_proxy.health_headers.Host=$hc_host"
+  echo -n " --label caddy.reverse_proxy.health_fails=$hc_attempts"
+  if [[ -n "$hc_host" ]] && fn-proxy-healthcheck-header-is-valid "Host" "$hc_host" && [[ "$hc_host" != *[[:space:]]* ]]; then
+    echo -n " --label caddy.reverse_proxy.health_headers.Host=$hc_host"
+  fi
+
+  while IFS=$'\t' read -r name value; do
+    [[ -z "$name" ]] && continue
+    if ! fn-caddy-healthcheck-header-name-is-valid "$name" || ! fn-proxy-healthcheck-header-is-valid "$name" "$value"; then
+      dokku_log_warn "Skipping invalid app.json healthcheck header for caddy: $name"
+      continue
+    fi
+
+    echo -n " --label 'caddy.reverse_proxy.health_headers.$name=\"$value\"'"
+  done < <(fn-proxy-get-healthcheck-headers "$healthcheck")
+
   echo -n " --label caddy.reverse_proxy.lb_try_duration=5s"
   echo -n " --label caddy.reverse_proxy.lb_try_interval=250ms"
 }
 
+fn-caddy-healthcheck-header-name-is-valid() {
+  declare desc="checks that a healthcheck header name can be used as a caddy label segment, which may not contain dots or end in an _<number> ordering suffix"
+  declare NAME="$1"
+
+  [[ "$NAME" == *.* ]] && return 1
+  [[ "$NAME" =~ _[0-9]+$ ]] && return 1
+  return 0
+}
+
 trigger-caddy-vhosts-docker-args-process-deploy "$@"

plugins/openresty-vhosts/docker-args-process-deploy

@@ -222,8 +222,7 @@ trigger-openresty-vhosts-docker-args-process-deploy() {
 fn-openresty-healthcheck-labels() {
   declare desc="outputs openresty upstream healthcheck and failover labels derived from the app's web readiness healthcheck"
   declare APP="$1"
-  local healthcheck hc_path hc_port hc_scheme hc_timeout hc_interval hc_attempts hc_host domain name value
+  local healthcheck hc_path hc_port hc_scheme hc_timeout hc_interval hc_attempts hc_host name value
-  local -a domains
 
   healthcheck="$(fn-proxy-get-readiness-healthcheck "$APP")"
   if [[ -z "$healthcheck" ]]; then
@@ -245,18 +244,7 @@ fn-openresty-healthcheck-labels() {
     dokku_log_fail "Invalid app.json healthcheck scheme for openresty: scheme must be one of http or https"
   fi
 
-  hc_host="$(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) == "host")) | .[0].value // empty')"
+  hc_host="$(fn-proxy-get-healthcheck-host "$APP" "$healthcheck")"
-  if [[ -z "$hc_host" ]]; then
-    read -r -d '' -a domains <<<"$(plugn trigger domains-list "$APP")" || true
-    for domain in "${domains[@]}"; do
-      if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then
-        continue
-      fi
-
-      hc_host="$domain"
-      break
-    done
-  fi
 
   echo -n " '--label=openresty.upstream-max-fails=1'"
   echo -n " '--label=openresty.upstream-fail-timeout=${hc_interval}s'"
@@ -267,28 +255,19 @@ fn-openresty-healthcheck-labels() {
   echo -n " '--label=openresty.healthcheck-timeout=$hc_timeout'"
   echo -n " '--label=openresty.healthcheck-interval=$hc_interval'"
   echo -n " '--label=openresty.healthcheck-fall=$hc_attempts'"
-  if [[ -n "$hc_host" ]] && fn-openresty-healthcheck-header-is-valid "Host" "$hc_host"; then
+  if [[ -n "$hc_host" ]] && fn-proxy-healthcheck-header-is-valid "Host" "$hc_host"; then
     echo -n " '--label=openresty.healthcheck-host=$hc_host'"
   fi
 
   while IFS=$'\t' read -r name value; do
     [[ -z "$name" ]] && continue
-    if ! fn-openresty-healthcheck-header-is-valid "$name" "$value"; then
+    if ! fn-proxy-healthcheck-header-is-valid "$name" "$value"; then
       dokku_log_warn "Skipping invalid app.json healthcheck header for openresty: $name"
       continue
     fi
 
     echo -n " '--label=openresty.healthcheck-header.$name=$value'"
-  done < <(echo "$healthcheck" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) != "host")) | .[] | [.name, .value] | @tsv')
+  done < <(fn-proxy-get-healthcheck-headers "$healthcheck")
-}
-
-fn-openresty-healthcheck-header-is-valid() {
-  declare desc="checks that a healthcheck header name is a token and its value is free of control, quote, and backslash characters"
-  declare NAME="$1" VALUE="$2"
-
-  [[ "$NAME" =~ ^[A-Za-z0-9!#$%\&*+.^_\`\|~-]+$ ]] || return 1
-  [[ "$VALUE" =~ [[:cntrl:]\"\'\\] ]] && return 1
-  return 0
 }
 
 trigger-openresty-vhosts-docker-args-process-deploy "$@"

plugins/proxy/functions

@@ -35,6 +35,45 @@ fn-proxy-healthcheck-path-is-valid() {
   return 0
 }
 
+fn-proxy-healthcheck-header-is-valid() {
+  declare desc="checks that an app.json healthcheck header name is a token and its value is free of control, quote, and backslash characters"
+  declare NAME="$1" VALUE="$2"
+
+  [[ "$NAME" =~ ^[A-Za-z0-9!#$%\&*+.^_\`\|~-]+$ ]] || return 1
+  [[ "$VALUE" =~ [[:cntrl:]\"\'\\] ]] && return 1
+  return 0
+}
+
+fn-proxy-get-healthcheck-host() {
+  declare desc="outputs the host header for a healthcheck, preferring an app.json Host header over the first app domain"
+  declare APP="$1" HEALTHCHECK="$2"
+  local domain hc_host
+  local -a domains
+
+  hc_host="$(echo "$HEALTHCHECK" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) == "host")) | .[0].value // empty')"
+  if [[ -n "$hc_host" ]]; then
+    echo "$hc_host"
+    return
+  fi
+
+  read -r -d '' -a domains <<<"$(plugn trigger domains-list "$APP")" || true
+  for domain in "${domains[@]}"; do
+    if [[ "$domain" == "localhost" ]] || [[ "$domain" == '*' ]]; then
+      continue
+    fi
+
+    echo "$domain"
+    return
+  done
+}
+
+fn-proxy-get-healthcheck-headers() {
+  declare desc="outputs the tab-separated name and value of each non-Host app.json healthcheck header"
+  declare HEALTHCHECK="$1"
+
+  echo "$HEALTHCHECK" | jq -r '.httpHeaders // [] | map(select((.name | ascii_downcase) != "host")) | .[] | [.name, .value] | @tsv'
+}
+
 fn-proxy-healthcheck-scheme-is-valid() {
   declare desc="checks that an app.json healthcheck scheme is either http or https"
   declare HC_SCHEME="$1"

tests/unit/caddy-vhosts-docker-args.bats

@@ -237,3 +237,75 @@ invoke_caddy_docker_args() {
   assert_failure
   assert_output_contains "Invalid app.json healthcheck path for caddy"
 }
+
+@test "(caddy-vhosts) readiness healthcheck maps attempts and headers" {
+  STUB_DOMAINS="example.com"
+  STUB_PORTS="http:80:5000"
+  STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","attempts":4,"httpHeaders":[{"name":"X-Check","value":"dokku check"},{"name":"host","value":"check.example.com"}]}]}}'
+
+  run invoke_caddy_docker_args
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "--label caddy.reverse_proxy.health_fails=4"
+  assert_output_contains "--label 'caddy.reverse_proxy.health_headers.X-Check=\"dokku check\"'"
+  assert_output_contains "--label caddy.reverse_proxy.health_headers.Host=check.example.com"
+  assert_output_contains "health_headers.Host=example.com" 0
+  assert_output_contains "health_headers.host" 0
+}
+
+@test "(caddy-vhosts) readiness healthcheck defaults attempts to 3" {
+  STUB_DOMAINS="example.com"
+  STUB_PORTS="http:80:5000"
+  STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/"}]}}'
+
+  run invoke_caddy_docker_args
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "--label caddy.reverse_proxy.health_fails=3"
+}
+
+@test "(caddy-vhosts) invalid healthcheck headers are skipped" {
+  STUB_DOMAINS="example.com"
+  STUB_PORTS="http:80:5000"
+  STUB_APP_JSON='{"healthchecks":{"web":[{"type":"readiness","path":"/","httpHeaders":[{"name":"X.Dotted","value":"a"},{"name":"X-Order_1","value":"a"},{"name":"X-Quote","value":"it'"'"'s"},{"name":"Bad Name","value":"a"},{"name":"X-Good","value":"ok"}]}]}}'
+
+  run invoke_caddy_docker_args
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Skipping invalid app.json healthcheck header for caddy" 4
+  assert_output_contains "health_headers.X-Good=" 1
+  assert_output_contains "health_headers.X.Dotted" 0
+  assert_output_contains "health_headers.X-Order_1" 0
+  assert_output_contains "health_headers.X-Quote" 0
+}
+
+@test "(caddy-vhosts) tls-internal emits caddy.tls=internal alongside https site labels" {
+  STUB_DOMAINS="example.com"
+  STUB_PORTS="$(printf 'http:80:5000\nhttps:443:5000')"
+  STUB_LETSENCRYPT_EMAIL="admin@example.com"
+  STUB_TLS_INTERNAL="true"
+
+  run invoke_caddy_docker_args
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "--label 'caddy=example.com'"
+  assert_output_contains "--label caddy.tls=internal"
+  assert_output_contains "caddy.reverse_proxy="
+}
+
+@test "(caddy-vhosts) tls-internal does not emit caddy.tls=internal for http-only sites" {
+  STUB_DOMAINS="example.com"
+  STUB_PORTS="http:80:5000"
+  STUB_TLS_INTERNAL="true"
+
+  run invoke_caddy_docker_args
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "--label 'caddy=example.com:80'"
+  assert_output_contains "caddy.tls=internal" 0
+}

tests/unit/caddy.bats

@@ -582,6 +582,47 @@ teardown() {
   assert_output_contains "Invalid app.json healthcheck path for caddy" -1
 }
 
+@test "(caddy) healthcheck headers from app.json" {
+  run /bin/bash -c "dokku proxy:set $TEST_APP caddy"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP setup_caddy_readiness_healthcheck_headers
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_headers.X-Check\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output '"dokku check"'
+
+  run /bin/bash -c "docker inspect $TEST_APP.web.1 --format '{{ index .Config.Labels \"caddy.reverse_proxy.health_fails\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "2"
+
+  sleep 5
+
+  run /bin/bash -c "docker exec caddy-caddy-1 cat /config/caddy/Caddyfile.autosave"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains 'X-Check "dokku check"'
+  assert_output_contains "health_fails 2"
+
+  run /bin/bash -c "docker logs caddy-caddy-1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Failed to convert caddyfile" 0
+
+  assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
+}
+
 setup_caddy_readiness_healthcheck() {
   local APP="$1"
   local APP_REPO_DIR="$2"
@@ -591,3 +632,13 @@ setup_caddy_readiness_healthcheck() {
   convert_to_dockerfile "$APP" "$APP_REPO_DIR"
   mv "$APP_REPO_DIR/app-readiness.json" "$APP_REPO_DIR/app.json"
 }
+
+setup_caddy_readiness_healthcheck_headers() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  convert_to_dockerfile "$APP" "$APP_REPO_DIR"
+  jq '.healthchecks.web[0].attempts = 2 | .healthchecks.web[0].httpHeaders = [{"name": "X-Check", "value": "dokku check"}]' "$APP_REPO_DIR/app-readiness.json" >"$APP_REPO_DIR/app.json"
+}