From 5241fd4be2efd8e420fe5210feb03ff7645d6bee Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 6 Nov 2026 20:43:16 -0500 Subject: [PATCH 01/10] feat: resolve nginx upstreams via coredns-docker Stops nginx from routing to stale container IPs after host reboots, Docker upgrades without live-restore, or runtime container restarts. The bundled `nginx.conf.sigil` can use an nginx `resolver` directive against the local `coredns-docker` socket and a variable-based `proxy_pass` keyed on `..docker`, so the upstream is resolved at request time honoring DNS TTL. `coredns-docker` becomes a hard `Depends:` of the dokku deb and is bundled in the Docker image under runit. Closes #8313. --- contrib/dependencies.json | 8 + debian/control | 2 +- .../runit/runsvdir/default/coredns-docker/run | 7 + docs/appendices/0.39.0-migration-guide.md | 16 ++ .../file-formats/nginx-conf-sigil.md | 51 ++++++- docs/getting-started/install/docker.md | 1 + docs/networking/proxies/nginx.md | 28 ++++ plugins/nginx-vhosts/functions | 20 ++- plugins/nginx-vhosts/internal-functions | 42 ++++++ plugins/nginx-vhosts/nginx_vhosts.go | 34 +++++ plugins/nginx-vhosts/report.go | 2 + .../src/nginx-property/nginx-property.go | 12 ++ plugins/nginx-vhosts/subcommands/set | 6 +- plugins/nginx-vhosts/template_test.go | 47 +++++- .../nginx-vhosts/templates/nginx.conf.sigil | 15 +- tests/unit/nginx-vhosts_17.bats | 139 ++++++++++++++++++ 16 files changed, 419 insertions(+), 11 deletions(-) create mode 100755 docker/etc/runit/runsvdir/default/coredns-docker/run create mode 100644 tests/unit/nginx-vhosts_17.bats diff --git a/contrib/dependencies.json b/contrib/dependencies.json index a2c7a3931..b4561830f 100644 --- a/contrib/dependencies.json +++ b/contrib/dependencies.json @@ -1,5 +1,13 @@ { "dependencies": [ + { + "name": "coredns-docker", + "version": "0.6.1", + "urls": { + "amd64": "https://github.com/dokku/coredns-docker/releases/download/v0.6.1/coredns-docker-linux-amd64", + "arm64": "https://github.com/dokku/coredns-docker/releases/download/v0.6.1/coredns-docker-linux-arm64" + } + }, { "name": "docker-container-healthchecker", "version": "0.16.1", diff --git a/debian/control b/debian/control index 703edb864..4d5e30a93 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.38.34 Section: web Priority: optional Architecture: amd64 -Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux +Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, coredns-docker, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux Recommends: herokuish, bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, jq, nginx (>= 1.8.0) | openresty, bind9-dnsutils, debconf, plugn, sudo, python3, rsyslog Maintainer: Jose Diaz-Gonzalez diff --git a/docker/etc/runit/runsvdir/default/coredns-docker/run b/docker/etc/runit/runsvdir/default/coredns-docker/run new file mode 100755 index 000000000..312b939d2 --- /dev/null +++ b/docker/etc/runit/runsvdir/default/coredns-docker/run @@ -0,0 +1,7 @@ +#!/bin/bash + +exec >>/var/log/services/coredns-docker +exec 2>&1 + +echo "Running coredns-docker" +exec /usr/bin/coredns-docker -conf /etc/coredns/Corefile diff --git a/docs/appendices/0.39.0-migration-guide.md b/docs/appendices/0.39.0-migration-guide.md index 420e5f7b4..4b40ebe05 100644 --- a/docs/appendices/0.39.0-migration-guide.md +++ b/docs/appendices/0.39.0-migration-guide.md @@ -7,6 +7,22 @@ - The `--buildpacks-stack`, `--buildpacks-global-stack`, and `--buildpacks-computed-stack` report flags (and the corresponding `stack`, `global-stack`, `computed-stack` JSON keys) have been removed from `buildpacks:report`. Use `builder-herokuish:report` and `builder-pack:report` instead. The `buildpacks:report` command continues to report the buildpacks `list`. - The internal `buildpack-stack-name` plugin trigger has been removed. Each builder now reads its own `stack` property directly during the build. - Changing a builder's `stack` now clears that builder's build cache so the next deploy starts fresh. For herokuish this removes the `cache-` Docker volume; for pack the next `pack build` runs with `--clear-cache`. +- The `dokku` Debian package now declares `coredns-docker (>= 0.6.0)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. +- The bundled `nginx.conf.sigil` now uses an nginx `resolver` directive and a variable-based `proxy_pass` against `..docker` instead of pinning the upstream to a literal IP. This is what addresses the long-standing post-reboot and post-`apt-upgrade-docker` stale-vhost bug ([#8313](https://github.com/dokku/dokku/issues/8313)) - the rendered upstream re-resolves at request time via coredns-docker, so a host reboot or single-container restart no longer leaves nginx routing to a stale IP. See the [nginx documentation](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for details. +- Two new global nginx properties control the resolver path: + + - `dns-resolver` (default: `127.0.0.1:1053`) - the address nginx talks to. + - `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. + + Both can be set via `dokku nginx:set --global `. +- The literal string `off` is the sentinel for disabling the resolver path. **Setting `dns-resolver` to the empty string does not disable it** - that deletes the property file and the new default (`127.0.0.1:1053`) re-engages. To revert to the legacy IP-based upstream block on hosts where coredns-docker cannot run (source/git installs, locally held coredns-docker package, postinst failures): + + ```shell + dokku nginx:set --global dns-resolver off + dokku proxy:build-config --all + ``` + +- Vendored `nginx.conf.sigil` templates are unaffected by the change. The existing `DOKKU_APP__LISTENERS` variable continues to be populated with the IP-based listener list exactly as before, so templates that only reference it continue to render IP-based upstreams regardless of the new property defaults. To opt a vendored template into the resolver path, switch its `proxy_pass` line to use the new `DOKKU_APP__LISTENER_HOST`, `NGINX_DNS_RESOLVER`, and `NGINX_DNS_ZONE` sigil variables - see the [nginx.conf.sigil reference](/docs/appendices/file-formats/nginx-conf-sigil.md#upstream-rendering-ip-based-vs-dns-based) for the exact snippet. ### Stack property migrated to builder plugins diff --git a/docs/appendices/file-formats/nginx-conf-sigil.md b/docs/appendices/file-formats/nginx-conf-sigil.md index 78fba3f58..bb9a063f6 100644 --- a/docs/appendices/file-formats/nginx-conf-sigil.md +++ b/docs/appendices/file-formats/nginx-conf-sigil.md @@ -4,7 +4,10 @@ The `nginx.conf.sigil` file is used to configure the nginx server for an applica ## Validation -A custom `nginx.conf.sigil` is pre-validated at the start of every deploy, immediately after it is extracted from the source tree and before the build phase runs. Pre-validation renders the template via sigil with the same parameters used at deploy time, wraps the rendered config in a minimal `events`/`http` scaffold, and runs `nginx -t` against the result. The deploy is aborted if either the sigil render or the `nginx -t` check fails, so build work is not wasted on a syntactically invalid template. When no app listeners exist yet (typical for first deploys), pre-validation injects a placeholder `127.0.0.1:5000` listener for `DOKKU_APP_WEB_LISTENERS` so that the rendered upstream block has a static server entry and `nginx -t` does not bail out on "host not found in upstream". +A custom `nginx.conf.sigil` is pre-validated at the start of every deploy, immediately after it is extracted from the source tree and before the build phase runs. Pre-validation renders the template via sigil with the same parameters used at deploy time, wraps the rendered config in a minimal `events`/`http` scaffold, and runs `nginx -t` against the result. The deploy is aborted if either the sigil render or the `nginx -t` check fails, so build work is not wasted on a syntactically invalid template. When no app listeners exist yet (typical for first deploys), pre-validation injects placeholders so the rendered config parses cleanly under `nginx -t`: + +- `DOKKU_APP_WEB_LISTENERS` is set to `127.0.0.1:5000` so the legacy `upstream` block has a static server entry. +- `DOKKU_APP_WEB_LISTENER_HOST` is set to `127.0.0.1` so the resolver-based `set $dokku_upstream "...";` line renders to a syntactically valid value. Pre-validation is skipped when the proxy type is not `nginx` or when `disable-custom-config` is set to `true` for the app. @@ -68,3 +71,49 @@ listen {{ if $.NGINX_BIND_ADDRESS_IP4 }}{{ $.NGINX_BIND_ADDRESS_IP4 }}:{{en ``` Dokku logs a deprecation warning during deploys when a custom template still uses the `listen ... http2` form, so the offending template can be located via the deploy output. + +## Upstream Rendering: IP-based vs DNS-based + +> [!IMPORTANT] +> New as of 0.39.0 + +The bundled `nginx.conf.sigil` supports two rendering paths for the upstream, and both sets of sigil variables are populated on every render. Vendored templates can use either path independently of the global `dns-resolver` setting. + +### IP-based upstream (legacy path) + +The classic form uses `DOKKU_APP__LISTENERS`: a space-separated `:` list, one entry per scaled instance of the process type. The IPs come from `/home/dokku//IP..` and the ports from `/home/dokku//PORT..`. A typical use is an `upstream` block with one `server :;` per instance: + +```nginx +upstream {{ $.APP }}-{{ $upstream_port }} { +{{ range $listeners := $.DOKKU_APP_WEB_LISTENERS | split " " }} +{{ $listener_list := $listeners | split ":" }} +{{ $listener_ip := index $listener_list 0 }} + server {{ $listener_ip }}:{{ $upstream_port }};{{ end }} +} +``` + +This path renders the IP that Dokku saw when the config was last built. After a host reboot, an `apt upgrade docker` without live-restore, or a single container restart, those IPs can become stale until `dokku proxy:build-config` runs again. The DNS-based path below addresses that staleness. + +### DNS-based upstream (resolver path) + +The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.6.0`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. + +The bundled template emits this snippet: + +```nginx +{{ if ne $.NGINX_DNS_RESOLVER "off" }} +resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; +set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; +proxy_pass http://$dokku_upstream; +{{ else }} +proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; +{{ end }} +``` + +The `$dokku_upstream` variable name is intentionally namespaced (rather than `$upstream`) so vendored templates that already define `$upstream` will not collide if they adopt the resolver branch. + +`NGINX_DNS_RESOLVER` defaults to `127.0.0.1:1053` and `NGINX_DNS_ZONE` defaults to `docker`; both are configurable via `dokku nginx:set --global dns-resolver ...` and `dokku nginx:set --global dns-zone ...`. The literal string `off` is the sentinel for disabling the resolver branch in the bundled template - empty strings are treated as "use the default" because the property is stored on disk and an empty value deletes the property file. + +### Choosing between the two + +The DNS-based path is the default for the bundled template. Vendored templates that have not been updated continue to use `DOKKU_APP__LISTENERS` and are unaffected by the global default. The DNS-based variables are populated regardless of whether `dns-resolver` is `off` or not, so a vendored template can adopt the resolver path on its own schedule by switching its rendering snippet, even if other apps on the host are still using IP-based upstreams. diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md index b336b21de..15d7efbe7 100644 --- a/docs/getting-started/install/docker.md +++ b/docs/getting-started/install/docker.md @@ -88,6 +88,7 @@ Dokku is run in the following configuration: - Image build cache is set to the data dir + `/home/dokku`. - The docker socket is mounted into container. - `DOKKU_HOST_ROOT` and `DOKKU_LIB_HOST_ROOT` tell Dokku how the host Docker daemon sees those same trees when creating bind mounts (app repos and plugin data such as named storage). +- `coredns-docker` (a CoreDNS plugin that serves DNS records for sibling Docker containers) is installed transitively via the dokku deb's `Depends:`, runs under runit on `127.0.0.1:1053` inside the container, and is what the `nginx-vhosts` `dns-resolver` default points at. This is what makes the `..docker` upstream resolution work out of the box - see the [nginx proxy doc](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for the operator-facing behavior. Application repositories, plugin config, as well as plugin data are persisted to disk within the specified host directory for `/var/lib/dokku`. `DOKKU_LIB_HOST_ROOT` must point at that host path's `var/lib/dokku` subtree so Docker `-v` mounts resolve under the data volume rather than under the host's own `/var/lib/dokku`. diff --git a/docs/networking/proxies/nginx.md b/docs/networking/proxies/nginx.md index 4ac343db6..8f5e0ab63 100644 --- a/docs/networking/proxies/nginx.md +++ b/docs/networking/proxies/nginx.md @@ -28,6 +28,34 @@ Nginx will proxy the requests in a [round-robin balancing fashion](http://nginx. > [!NOTE] > Due to how the plugin is implemented, if an app successfully starts up `web` containers but fails to deploy some other containers, nginx may eventually stop routing requests. Users should revert their code in these cases, or manually trigger `dokku proxy:build-config $APP` in order to ensure requests route to the new web containers. +### Upstream Resolution via coredns-docker + +> [!IMPORTANT] +> New as of 0.39.0 + +Dokku resolves nginx upstreams through [coredns-docker](https://github.com/dokku/coredns-docker), a CoreDNS plugin that watches the Docker daemon and serves fresh `A` records keyed off container labels. The nginx `resolver` directive points at coredns-docker's local socket (`127.0.0.1:1053`) and the rendered upstream is a variable-based `proxy_pass`, which forces nginx to re-resolve at request time honoring DNS TTL. This is how nginx avoids serving the wrong vhost after a host reboot, after `apt upgrade docker`, or after a single container restarts mid-runtime: the on-disk nginx config no longer pins a stale IP. + +`coredns-docker` (>= `0.6.0`) is installed automatically as a transitive `Depends:` of the `dokku` Debian package, so `apt install dokku` and `apt upgrade dokku` pull it in without operator action. The shipped `/etc/coredns/Corefile` already binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from Dokku's existing `com.dokku.app-name` and `com.dokku.process-type` container labels. Multiple containers for the same process type collapse onto one multi-A record set, so scaled apps load-balance across instances via DNS without any per-container configuration. The bundled Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. + +No `systemd-resolved` or `/etc/resolv.conf` change is required - nginx talks to `coredns-docker` directly via the `resolver` directive in each app's `nginx.conf`. + +The behavior is controlled by two global nginx properties: + +- `dns-resolver` (default: `127.0.0.1:1053`) - the resolver address nginx talks to. +- `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. + +To revert to the legacy IP-based upstream block (for instance on a host where `coredns-docker` cannot be installed): + +```shell +dokku nginx:set --global dns-resolver off +dokku proxy:build-config --all +``` + +> [!NOTE] +> The sentinel for disabling the feature is the literal string `off`. Setting `dns-resolver` to the empty string (`""`) does **not** disable it - that deletes the property and the default (`127.0.0.1:1053`) re-engages. Use `off` to disable, the default to re-enable. + +See [the `nginx.conf.sigil` reference](/docs/appendices/file-formats/nginx-conf-sigil.md) for the sigil variables that drive the resolver-based render, and how vendored templates can adopt or skip it. + ### Nginx Configuration for Undeployed Apps > [!IMPORTANT] diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index 6426c4b72..051a8f525 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -324,6 +324,8 @@ fn-nginx-vhosts-render-template() { local NGINX_BIND_ADDRESS_IP4="$(fn-nginx-computed-bind-address-ipv4 "$APP")" local NGINX_BIND_ADDRESS_IP6="$(fn-nginx-computed-bind-address-ipv6 "$APP")" + local NGINX_DNS_RESOLVER="$(fn-nginx-computed-dns-resolver "$APP")" + local NGINX_DNS_ZONE="$(fn-nginx-computed-dns-zone "$APP")" local NGINX_UNDERSCORE_IN_HEADERS="$(fn-nginx-computed-underscore-in-headers "$APP")" local PROXY_X_FORWARDED_FOR="$(fn-nginx-computed-x-forwarded-for-value "$APP")" local PROXY_X_FORWARDED_PORT="$(fn-nginx-computed-x-forwarded-port-value "$APP")" @@ -354,6 +356,8 @@ fn-nginx-vhosts-render-template() { NGINX_ERROR_LOG_PATH="$NGINX_ERROR_LOG_PATH" NGINX_BIND_ADDRESS_IP4="$NGINX_BIND_ADDRESS_IP4" NGINX_BIND_ADDRESS_IP6="$NGINX_BIND_ADDRESS_IP6" + NGINX_DNS_RESOLVER="$NGINX_DNS_RESOLVER" + NGINX_DNS_ZONE="$NGINX_DNS_ZONE" HTTP2_DIRECTIVE_SUPPORTED="$HTTP2_DIRECTIVE_SUPPORTED" HTTP2_PUSH_SUPPORTED="$HTTP2_PUSH_SUPPORTED" GRPC_SUPPORTED="$GRPC_SUPPORTED" @@ -386,21 +390,29 @@ fn-nginx-vhosts-render-template() { PROXY_X_FORWARDED_SSL="$PROXY_X_FORWARDED_SSL") if [[ "$IS_DEPLOYED_WITH_LISTENERS" == "true" ]]; then - local line PROC_TYPE LISTENERS UPP_PROC_TYPE + local line PROC_TYPE LISTENERS UPP_PROC_TYPE LISTENER_HOST while read -r line || [[ -n "$line" ]]; do PROC_TYPE=${line%%=*} LISTENERS="$(plugn trigger network-get-listeners "$APP" "$PROC_TYPE" | xargs)" UPP_PROC_TYPE="${PROC_TYPE^^}" UPP_PROC_TYPE="${UPP_PROC_TYPE//-/_}" SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENERS=$LISTENERS") + LISTENER_HOST="" + if [[ -n "$NGINX_DNS_ZONE" ]]; then + LISTENER_HOST="${APP}.${PROC_TYPE}.${NGINX_DNS_ZONE}" + fi + SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENER_HOST=$LISTENER_HOST") done < <(plugn trigger ps-current-scale "$APP") elif [[ "$PRE_VALIDATE" == "true" ]]; then - # During pre-validation no listeners exist yet; supply a placeholder so - # any upstream block in the template renders with a static server entry - # and nginx -t does not bail out on "host not found in upstream". + # During pre-validation no listeners exist yet; supply placeholders so + # both the legacy IP-based upstream block and the DNS-based resolver + # branch render syntactically valid nginx config that nginx -t can + # parse (no "host not found in upstream" and no empty set $var). SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENERS="127.0.0.1:5000") + SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENER_HOST="127.0.0.1") else SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENERS="") + SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENER_HOST="") fi sigil "${SIGIL_PARAMS[@]}" | cat -s >"$OUTPUT_PATH" diff --git a/plugins/nginx-vhosts/internal-functions b/plugins/nginx-vhosts/internal-functions index a2616eb84..3b9e378eb 100755 --- a/plugins/nginx-vhosts/internal-functions +++ b/plugins/nginx-vhosts/internal-functions @@ -172,6 +172,48 @@ fn-nginx-global-disable-custom-config() { fn-get-property --app "$APP" --global "disable-custom-config" } +fn-nginx-dns-resolver() { + declare desc="get the configured dns-resolver value" + declare APP="$1" + + fn-get-property --app "$APP" "dns-resolver" +} + +fn-nginx-computed-dns-resolver() { + declare desc="get the computed dns-resolver value" + declare APP="$1" + + fn-get-property --app "$APP" --computed "dns-resolver" +} + +fn-nginx-global-dns-resolver() { + declare desc="get the global dns-resolver value" + declare APP="$1" + + fn-get-property --app "$APP" --global "dns-resolver" +} + +fn-nginx-dns-zone() { + declare desc="get the configured dns-zone value" + declare APP="$1" + + fn-get-property --app "$APP" "dns-zone" +} + +fn-nginx-computed-dns-zone() { + declare desc="get the computed dns-zone value" + declare APP="$1" + + fn-get-property --app "$APP" --computed "dns-zone" +} + +fn-nginx-global-dns-zone() { + declare desc="get the global dns-zone value" + declare APP="$1" + + fn-get-property --app "$APP" --global "dns-zone" +} + fn-nginx-error-log-path() { declare desc="get the configured error log path" declare APP="$1" diff --git a/plugins/nginx-vhosts/nginx_vhosts.go b/plugins/nginx-vhosts/nginx_vhosts.go index 7a51f6e10..bd6b14cae 100644 --- a/plugins/nginx-vhosts/nginx_vhosts.go +++ b/plugins/nginx-vhosts/nginx_vhosts.go @@ -150,6 +150,40 @@ func AppDisableCustomConfig(appName string) string { return common.PropertyGet("nginx", appName, "disable-custom-config") } +func AppDnsResolver(appName string) string { + return common.PropertyGet("nginx", appName, "dns-resolver") +} + +func ComputedDnsResolver(appName string) string { + appValue := AppDnsResolver(appName) + if appValue != "" { + return appValue + } + + return GlobalDnsResolver() +} + +func GlobalDnsResolver() string { + return common.PropertyGetDefault("nginx", "--global", "dns-resolver", "127.0.0.1:1053") +} + +func AppDnsZone(appName string) string { + return common.PropertyGet("nginx", appName, "dns-zone") +} + +func ComputedDnsZone(appName string) string { + appValue := AppDnsZone(appName) + if appValue != "" { + return appValue + } + + return GlobalDnsZone() +} + +func GlobalDnsZone() string { + return common.PropertyGetDefault("nginx", "--global", "dns-zone", "docker") +} + func ComputedDisableCustomConfig(appName string) string { appValue := AppDisableCustomConfig(appName) if appValue != "" { diff --git a/plugins/nginx-vhosts/report.go b/plugins/nginx-vhosts/report.go index 3cf18eb3c..f6d29bd68 100644 --- a/plugins/nginx-vhosts/report.go +++ b/plugins/nginx-vhosts/report.go @@ -29,6 +29,8 @@ func nginxReportProps() []nginxReportProp { {"client-header-timeout", AppClientHeaderTimeout, ComputedClientHeaderTimeout, wrapGlobal(GlobalClientHeaderTimeout)}, {"client-max-body-size", AppClientMaxBodySize, ComputedClientMaxBodySize, wrapGlobal(GlobalClientMaxBodySize)}, {"disable-custom-config", AppDisableCustomConfig, ComputedDisableCustomConfig, wrapGlobal(GlobalDisableCustomConfig)}, + {"dns-resolver", AppDnsResolver, ComputedDnsResolver, wrapGlobal(GlobalDnsResolver)}, + {"dns-zone", AppDnsZone, ComputedDnsZone, wrapGlobal(GlobalDnsZone)}, {"error-log-path", AppErrorLogPath, ComputedErrorLogPath, wrapGlobal(GlobalErrorLogPath)}, {"hsts-include-subdomains", AppHSTSIncludeSubdomains, ComputedHSTSIncludeSubdomains, wrapGlobal(GlobalHSTSIncludeSubdomains)}, {"hsts-max-age", AppHSTSMaxAge, ComputedHSTSMaxAge, wrapGlobal(GlobalHSTSMaxAge)}, diff --git a/plugins/nginx-vhosts/src/nginx-property/nginx-property.go b/plugins/nginx-vhosts/src/nginx-property/nginx-property.go index 98b719b4d..bd1e3c28b 100644 --- a/plugins/nginx-vhosts/src/nginx-property/nginx-property.go +++ b/plugins/nginx-vhosts/src/nginx-property/nginx-property.go @@ -51,6 +51,10 @@ func appValue(appName string, property string) string { value = nginx_vhosts.AppClientHeaderTimeout(appName) case "disable-custom-config": value = nginx_vhosts.AppDisableCustomConfig(appName) + case "dns-resolver": + value = nginx_vhosts.AppDnsResolver(appName) + case "dns-zone": + value = nginx_vhosts.AppDnsZone(appName) case "error-log-path": value = nginx_vhosts.AppErrorLogPath(appName) case "hsts-include-subdomains": @@ -123,6 +127,10 @@ func computedValue(appName string, property string) string { value = nginx_vhosts.ComputedClientMaxBodySize(appName) case "disable-custom-config": value = nginx_vhosts.ComputedDisableCustomConfig(appName) + case "dns-resolver": + value = nginx_vhosts.ComputedDnsResolver(appName) + case "dns-zone": + value = nginx_vhosts.ComputedDnsZone(appName) case "error-log-path": value = nginx_vhosts.ComputedErrorLogPath(appName) case "hsts-include-subdomains": @@ -195,6 +203,10 @@ func globalValue(appName string, property string) string { value = nginx_vhosts.GlobalClientMaxBodySize() case "disable-custom-config": value = nginx_vhosts.GlobalDisableCustomConfig() + case "dns-resolver": + value = nginx_vhosts.GlobalDnsResolver() + case "dns-zone": + value = nginx_vhosts.GlobalDnsZone() case "error-log-path": value = nginx_vhosts.GlobalErrorLogPath() case "hsts-include-subdomains": diff --git a/plugins/nginx-vhosts/subcommands/set b/plugins/nginx-vhosts/subcommands/set index ee8ad3530..439d421d4 100755 --- a/plugins/nginx-vhosts/subcommands/set +++ b/plugins/nginx-vhosts/subcommands/set @@ -9,13 +9,13 @@ cmd-nginx-set() { declare cmd="nginx:set" [[ "$1" == "$cmd" ]] && shift 1 declare APP="$1" KEY="$2" VALUE="$3" - local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") - local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") + local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") + local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") [[ -z "$KEY" ]] && dokku_log_fail "No key specified" if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then - dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl" + dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, dns-resolver, dns-zone, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl" fi if ! fn-in-array "$KEY" "${GLOBAL_KEYS[@]}" && [[ "$APP" == "--global" ]]; then diff --git a/plugins/nginx-vhosts/template_test.go b/plugins/nginx-vhosts/template_test.go index 4248e5532..ebfb02cea 100644 --- a/plugins/nginx-vhosts/template_test.go +++ b/plugins/nginx-vhosts/template_test.go @@ -30,6 +30,7 @@ func defaultVars() map[string]interface{} { "SSL_INUSE": "", "APP_SSL_PATH": "/home/dokku/app/tls", "DOKKU_APP_WEB_LISTENERS": "127.0.0.1:5000", + "DOKKU_APP_WEB_LISTENER_HOST": "app.web.docker", "PROXY_PORT_MAP": "http:80:5000", "PROXY_UPSTREAM_PORTS": "5000", "PROXY_PORT": "80", @@ -37,6 +38,8 @@ func defaultVars() map[string]interface{} { "PROXY_KEEPALIVE": "", "NGINX_BIND_ADDRESS_IP4": "", "NGINX_BIND_ADDRESS_IP6": "::", + "NGINX_DNS_RESOLVER": "127.0.0.1:1053", + "NGINX_DNS_ZONE": "docker", "NGINX_ACCESS_LOG_PATH": "/var/log/nginx/app-access.log", "NGINX_ACCESS_LOG_FORMAT": "", "NGINX_ERROR_LOG_PATH": "/var/log/nginx/app-error.log", @@ -95,7 +98,10 @@ func mustNotContain(t *testing.T, out, needle string) { func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { out := renderTemplate(t, defaultVars()) mustContain(t, out, "listen [::]:80;") - mustContain(t, out, "proxy_pass http://app-5000;") + mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;") + mustContain(t, out, `set $dokku_upstream "app.web.docker:5000";`) + mustContain(t, out, "proxy_pass http://$dokku_upstream;") + mustNotContain(t, out, "upstream app-5000 {") mustContain(t, out, "error_page 500 501 502 503") mustContain(t, out, "server_name app.example.com;") mustNotContain(t, out, "ssl_certificate") @@ -103,6 +109,17 @@ func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { mustNotContain(t, out, "http2_push_preload") } +func TestTemplate_HTTPOnlyBasicProxyWithResolverOff(t *testing.T) { + v := defaultVars() + v["NGINX_DNS_RESOLVER"] = "off" + out := renderTemplate(t, v) + mustContain(t, out, "listen [::]:80;") + mustContain(t, out, "proxy_pass http://app-5000;") + mustContain(t, out, "upstream app-5000 {") + mustNotContain(t, out, "resolver 127.0.0.1:1053") + mustNotContain(t, out, "$dokku_upstream") +} + func TestTemplate_HTTPSEmitsPushPreloadWhenSupported(t *testing.T) { v := defaultVars() v["PROXY_PORT_MAP"] = "https:443:5000" @@ -128,6 +145,7 @@ func TestTemplate_HTTPRedirectsToHTTPSWhenSSLInUse(t *testing.T) { v["PROXY_PORT_MAP"] = "http:80:5000 https:443:5000" v["SSL_INUSE"] = "true" v["SSL_SERVER_NAME"] = "app.example.com" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) port80, _, ok := strings.Cut(out, "listen [::]:443") @@ -170,6 +188,7 @@ func TestTemplate_NoWebListenersReturns502(t *testing.T) { v["DOKKU_APP_WEB_LISTENERS"] = "" out := renderTemplate(t, v) mustContain(t, out, "return 502;") + mustNotContain(t, out, "proxy_pass http://$dokku_upstream;") mustNotContain(t, out, "proxy_pass http://app-5000;") } @@ -177,18 +196,30 @@ func TestTemplate_GRPCNoSSL(t *testing.T) { v := defaultVars() v["PROXY_PORT_MAP"] = "grpc:50051:50051" v["PROXY_UPSTREAM_PORTS"] = "50051" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "grpc_pass grpc://app-50051;") mustContain(t, out, "http2") mustNotContain(t, out, "ssl_certificate") } +func TestTemplate_GRPCNoSSLWithResolver(t *testing.T) { + v := defaultVars() + v["PROXY_PORT_MAP"] = "grpc:50051:50051" + v["PROXY_UPSTREAM_PORTS"] = "50051" + out := renderTemplate(t, v) + mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;") + mustContain(t, out, `set $dokku_upstream "app.web.docker:50051";`) + mustContain(t, out, "grpc_pass grpc://$dokku_upstream;") +} + func TestTemplate_GRPCS(t *testing.T) { v := defaultVars() v["PROXY_PORT_MAP"] = "grpcs:443:50051" v["PROXY_UPSTREAM_PORTS"] = "50051" v["SSL_INUSE"] = "true" v["SSL_SERVER_NAME"] = "app.example.com" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "ssl_certificate /home/dokku/app/tls/server.crt;") mustContain(t, out, "grpc_pass grpc://app-50051;") @@ -221,6 +252,7 @@ func TestTemplate_UpstreamBlock(t *testing.T) { v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001" v["PROXY_UPSTREAM_PORTS"] = "5000 5001" v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "upstream app-5000 {") mustContain(t, out, "upstream app-5001 {") @@ -230,13 +262,26 @@ func TestTemplate_UpstreamBlock(t *testing.T) { mustContain(t, out, "server 10.0.0.2:5001;") } +func TestTemplate_UpstreamBlockOmittedInResolverMode(t *testing.T) { + v := defaultVars() + v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001" + v["PROXY_UPSTREAM_PORTS"] = "5000 5001" + v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000" + out := renderTemplate(t, v) + mustNotContain(t, out, "upstream app-5000 {") + mustNotContain(t, out, "upstream app-5001 {") + mustNotContain(t, out, "server 10.0.0.1:5000;") +} + func TestTemplate_UpstreamWithKeepalive(t *testing.T) { v := defaultVars() v["PROXY_KEEPALIVE"] = "16" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "keepalive 16;") v2 := defaultVars() + v2["NGINX_DNS_RESOLVER"] = "off" out2 := renderTemplate(t, v2) mustNotContain(t, out2, "keepalive 16;") } diff --git a/plugins/nginx-vhosts/templates/nginx.conf.sigil b/plugins/nginx-vhosts/templates/nginx.conf.sigil index e43760b6f..57e6ec3c4 100644 --- a/plugins/nginx-vhosts/templates/nginx.conf.sigil +++ b/plugins/nginx-vhosts/templates/nginx.conf.sigil @@ -84,7 +84,13 @@ server { gzip_vary on; gzip_comp_level 6; +{{ if ne $.NGINX_DNS_RESOLVER "off" }} + resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; + set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; + proxy_pass http://$dokku_upstream; +{{ else }} proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; +{{ end }} {{ if and $is_ssl (eq $.HTTP2_PUSH_SUPPORTED "true") }}http2_push_preload on; {{ end }} proxy_http_version {{ $.PROXY_HTTP_VERSION }}; proxy_connect_timeout {{ $.PROXY_CONNECT_TIMEOUT }}; @@ -166,7 +172,13 @@ server { {{ end }} location / { +{{ if ne $.NGINX_DNS_RESOLVER "off" }} + resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; + set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; + grpc_pass grpc://$dokku_upstream; +{{ else }} grpc_pass grpc://{{ $.APP }}-{{ $upstream_port }}; +{{ end }} } client_max_body_size {{ $.CLIENT_MAX_BODY_SIZE }}; @@ -176,6 +188,7 @@ server { {{ end }} {{ end }} +{{ if eq $.NGINX_DNS_RESOLVER "off" }} {{ if $.DOKKU_APP_WEB_LISTENERS }} {{ range $upstream_port := $.PROXY_UPSTREAM_PORTS | split " " }} upstream {{ $.APP }}-{{ $upstream_port }} { @@ -185,7 +198,7 @@ upstream {{ $.APP }}-{{ $upstream_port }} { {{ $listener_ip := index $listener_list 0 }} server {{ $listener_ip }}:{{ $upstream_port }};{{ end }} } -{{ end }}{{ end }} +{{ end }}{{ end }}{{ end }} {{ range $entry := $.PROXY_ROUTE_LISTENERS | split " " }} {{ if $entry }} diff --git a/tests/unit/nginx-vhosts_17.bats b/tests/unit/nginx-vhosts_17.bats new file mode 100644 index 000000000..01c67e7c8 --- /dev/null +++ b/tests/unit/nginx-vhosts_17.bats @@ -0,0 +1,139 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup + [[ -f "$DOKKU_ROOT/VHOST" ]] && cp -fp "$DOKKU_ROOT/VHOST" "$DOKKU_ROOT/VHOST.bak" + create_app +} + +teardown() { + dokku --quiet nginx:set --global dns-resolver + dokku --quiet nginx:set --global dns-zone + destroy_app + [[ -f "$DOKKU_ROOT/VHOST.bak" ]] && mv "$DOKKU_ROOT/VHOST.bak" "$DOKKU_ROOT/VHOST" && chown dokku:dokku "$DOKKU_ROOT/VHOST" + global_teardown +} + +@test "(nginx-vhosts:set) dns-resolver and dns-zone defaults report 127.0.0.1:1053 and docker" { + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-resolver" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "127.0.0.1:1053" + + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-zone" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker" +} + +@test "(nginx-vhosts:set) dns-resolver and dns-zone can be overridden globally" { + run /bin/bash -c "dokku nginx:set --global dns-resolver 127.0.0.2:5353" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:set --global dns-zone test" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-resolver" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "127.0.0.2:5353" + + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-zone" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "test" +} + +@test "(nginx-vhosts) generated nginx.conf uses resolver and variable-based proxy_pass by default" { + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "resolver 127.0.0.1:1053 valid=10s ipv6=off" + assert_output_contains "set \$dokku_upstream" + assert_output_contains "${TEST_APP}.web.docker:" + assert_output_contains "proxy_pass http://\$dokku_upstream" + assert_output_not_contains "upstream ${TEST_APP}-" +} + +@test "(nginx-vhosts) generated nginx.conf falls back to IP-based upstream when dns-resolver=off" { + run /bin/bash -c "dokku nginx:set --global dns-resolver off" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_contains "resolver 127.0.0.1:1053" + assert_output_not_contains "\$dokku_upstream" + assert_output_contains "upstream ${TEST_APP}-" +} + +@test "(nginx-vhosts) setting dns-resolver to empty re-engages the default resolver path" { + run /bin/bash -c "dokku nginx:set --global dns-resolver off" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + assert_output_contains "upstream ${TEST_APP}-" + + run /bin/bash -c "dokku nginx:set --global dns-resolver" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku proxy:build-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "resolver 127.0.0.1:1053 valid=10s ipv6=off" + assert_output_not_contains "upstream ${TEST_APP}-" +} + +@test "(nginx-vhosts) vendored nginx.conf.sigil renders IP-based upstream regardless of dns-resolver" { + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP custom_nginx_template + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_contains "resolver 127.0.0.1:1053" + assert_output_not_contains "\$dokku_upstream" + assert_output_contains "upstream ${TEST_APP}-" +} From 599bc7b243e044f47dc8e116ed1ff8b717faee68 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 May 2026 00:54:43 -0400 Subject: [PATCH 02/10] ci: install coredns-docker before unpacking dokku.deb The dokku deb now declares `coredns-docker (>= 0.6.1)` as a hard `Depends:`, but `tests/ci/setup.sh` uses `dpkg -i` (not `apt-get install`) so transitive deps are not resolved and the install fails. Pre-download the `coredns-docker` deb from packagecloud and include it in the same `dpkg -i` invocation that already installs `docker-container-healthchecker`, `docker-image-labeler`, `lambda-builder`, and the other dokku-maintained deps. --- tests/ci/setup.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index e07edab5d..84cc019ed 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -20,6 +20,12 @@ install_dependencies() { curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/noble/docker-container-healthchecker_${DOCKER_CONTAINER_HEALTHCHECKER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${DOCKER_CONTAINER_HEALTHCHECKER_PACKAGE_NAME}" fi + COREDNS_DOCKER_VERSION=$(jq -r --arg name coredns-docker '.dependencies[] | select(.name == $name) | .version' contrib/dependencies.json) + COREDNS_DOCKER_PACKAGE_NAME="coredns-docker_${COREDNS_DOCKER_VERSION}_amd64.deb" + if [[ ! -f "$ROOT_DIR/build/${COREDNS_DOCKER_PACKAGE_NAME}" ]]; then + curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/noble/coredns-docker_${COREDNS_DOCKER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${COREDNS_DOCKER_PACKAGE_NAME}" + fi + HEROKUISH_VERSION=$(jq -r --arg name herokuish '.recommendations[] | select(.name == $name) | .version' contrib/dependencies.json) HEROKUISH_PACKAGE_NAME="herokuish_${HEROKUISH_VERSION}_amd64.deb" if [[ ! -f "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}" ]]; then @@ -68,6 +74,7 @@ install_dependencies() { ls -lah "${ROOT_DIR}/build/" sudo dpkg -i \ + "${ROOT_DIR}/build/$COREDNS_DOCKER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$DOCKER_CONTAINER_HEALTHCHECKER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ From ea72a80139b6d9df5980480af9279f34eb5f564d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 May 2026 01:05:07 -0400 Subject: [PATCH 03/10] test: verify nginx returns 502 when coredns-docker is unavailable Adds bats coverage on the bare-metal CI host (stops `coredns-docker` via `systemctl` and asserts the rendered vhost returns 502, then restarts and asserts recovery to 200) and a parallel `tests/test_coredns_outage` driver wired into `make test-ci-docker` that exercises the same flow against the dokku container using `sv stop|start coredns-docker`. Answers the open thread on whether the resolver-mode upstream surfaces a useful error when the DNS service is down. --- tests.mk | 6 +- tests/test_coredns_outage | 106 ++++++++++++++++++++++++++++++++ tests/unit/nginx-vhosts_17.bats | 25 ++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100755 tests/test_coredns_outage diff --git a/tests.mk b/tests.mk index da1230b1a..b0e96dd57 100644 --- a/tests.mk +++ b/tests.mk @@ -239,6 +239,10 @@ deploy-test-config: @echo deploying config app... cd tests && ./test_deploy ./apps/config $(DOKKU_DOMAIN) +deploy-test-coredns-docker-outage: + @echo testing nginx-vhosts behavior when coredns-docker is unavailable... + cd tests && ./test_coredns_outage $(DOKKU_DOMAIN) docker + deploy-test-dockerfile: @echo deploying dockerfile app... cd tests && ./test_deploy ./apps/dockerfile $(DOKKU_DOMAIN) @@ -342,7 +346,7 @@ deploy-test-storage-lib-host-root: @echo deploy-test-storage-lib-host-root cd tests/docker && bats storage.bats -test-ci-docker: setup-docker-deploy-tests deploy-test-checks-root deploy-test-config deploy-test-multi deploy-test-go-fail-predeploy deploy-test-go-fail-postdeploy deploy-test-storage-lib-host-root test-ci-binaries +test-ci-docker: setup-docker-deploy-tests deploy-test-checks-root deploy-test-config deploy-test-multi deploy-test-go-fail-predeploy deploy-test-go-fail-postdeploy deploy-test-storage-lib-host-root deploy-test-coredns-docker-outage test-ci-binaries generate-ssl-tars: generate-ssl-tar generate-ssl-sans-tar generate-ssl-wildcard-tar diff --git a/tests/test_coredns_outage b/tests/test_coredns_outage new file mode 100755 index 000000000..6babf416f --- /dev/null +++ b/tests/test_coredns_outage @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -xeo pipefail + +SELF=$(command -v "$0") +TARGET="$1" +SCOPE="${2:-host}" +REMOTE="dokku@$TARGET" +APP="test-coredns-outage-$RANDOM" + +cleanup() { + local exit_code=$? + set +e + start_coredns_docker + ssh "$REMOTE" apps:destroy "$APP" --force >/dev/null 2>&1 + ssh "$REMOTE" events:off >/dev/null 2>&1 + exit $exit_code +} + +stop_coredns_docker() { + case "$SCOPE" in + docker) + docker exec dokku sv stop coredns-docker + ;; + host) + sudo systemctl stop coredns-docker + ;; + *) + echo "Unknown scope: $SCOPE" + exit 1 + ;; + esac +} + +start_coredns_docker() { + case "$SCOPE" in + docker) + docker exec dokku sv start coredns-docker + ;; + host) + sudo systemctl start coredns-docker + ;; + *) + echo "Unknown scope: $SCOPE" + exit 1 + ;; + esac +} + +curl_status() { + curl -kSso /dev/null -w "%{http_code}" -m 5 "http://$APP.$TARGET/" +} + +wait_for_status() { + local expected="$1" attempts="${2:-15}" actual + while [[ $attempts -gt 0 ]]; do + actual="$(curl_status || true)" + [[ "$actual" == "$expected" ]] && return 0 + attempts=$((attempts - 1)) + sleep 1 + done + echo "Expected HTTP $expected, last observed $actual" + return 1 +} + +trap cleanup INT TERM EXIT + +ssh "$REMOTE" events:on + +TMP=$(mktemp -d "/tmp/$TARGET.XXXXX") +rmdir "$TMP" && cp -r "$(dirname "$SELF")/apps/python" "$TMP" + +pushd "$TMP" >/dev/null +git init -q +git config user.email "robot@example.com" +git config user.name "Test Robot" +git remote add target "${REMOTE}:${APP}" +[[ -f gitignore ]] && mv gitignore .gitignore +git add . +git commit -qm 'initial commit' +git push target master +popd >/dev/null +rm -rf "$TMP" + +echo "-----> Verifying app responds before outage" +wait_for_status 200 + +echo "-----> Stopping coredns-docker ($SCOPE)" +stop_coredns_docker + +# Wait long enough for any cached DNS entry (resolver valid=10s in the template) +sleep 12 + +echo "-----> Verifying nginx returns 502 with coredns-docker stopped" +actual="$(curl_status || true)" +if [[ "$actual" != "502" ]]; then + echo "Expected HTTP 502 with coredns-docker stopped, observed $actual" + exit 1 +fi + +echo "-----> Restarting coredns-docker" +start_coredns_docker + +echo "-----> Verifying app recovers" +wait_for_status 200 + +echo "-----> coredns-docker outage test PASSED" diff --git a/tests/unit/nginx-vhosts_17.bats b/tests/unit/nginx-vhosts_17.bats index 01c67e7c8..6f51c6356 100644 --- a/tests/unit/nginx-vhosts_17.bats +++ b/tests/unit/nginx-vhosts_17.bats @@ -9,6 +9,7 @@ setup() { } teardown() { + sudo systemctl start coredns-docker >/dev/null 2>&1 || true dokku --quiet nginx:set --global dns-resolver dokku --quiet nginx:set --global dns-zone destroy_app @@ -137,3 +138,27 @@ teardown() { assert_output_not_contains "\$dokku_upstream" assert_output_contains "upstream ${TEST_APP}-" } + +@test "(nginx-vhosts) nginx returns 502 when coredns-docker is stopped" { + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run sudo systemctl stop coredns-docker + echo "output: $output" + echo "status: $status" + assert_success + + run curl --connect-to "$TEST_APP.$DOKKU_DOMAIN:80:localhost:80" -kSso /dev/null -w "%{http_code}" -m 5 "http://$TEST_APP.$DOKKU_DOMAIN/" + echo "output: $output" + echo "status: $status" + assert_output "502" + + run sudo systemctl start coredns-docker + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response http "$TEST_APP.$DOKKU_DOMAIN" 80 / "" 200 +} From 1d2854fa933aa8d288aa92f2ede15ab0c2f9029a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 May 2026 11:15:56 -0400 Subject: [PATCH 04/10] chore: run shfmt --- tests/test_coredns_outage | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/test_coredns_outage b/tests/test_coredns_outage index 6babf416f..f1a168b01 100755 --- a/tests/test_coredns_outage +++ b/tests/test_coredns_outage @@ -18,31 +18,31 @@ cleanup() { stop_coredns_docker() { case "$SCOPE" in - docker) - docker exec dokku sv stop coredns-docker - ;; - host) - sudo systemctl stop coredns-docker - ;; - *) - echo "Unknown scope: $SCOPE" - exit 1 - ;; + docker) + docker exec dokku sv stop coredns-docker + ;; + host) + sudo systemctl stop coredns-docker + ;; + *) + echo "Unknown scope: $SCOPE" + exit 1 + ;; esac } start_coredns_docker() { case "$SCOPE" in - docker) - docker exec dokku sv start coredns-docker - ;; - host) - sudo systemctl start coredns-docker - ;; - *) - echo "Unknown scope: $SCOPE" - exit 1 - ;; + docker) + docker exec dokku sv start coredns-docker + ;; + host) + sudo systemctl start coredns-docker + ;; + *) + echo "Unknown scope: $SCOPE" + exit 1 + ;; esac } From b5315de49b4c57a8ab93d73301773e79eabe44f2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 May 2026 12:01:21 -0400 Subject: [PATCH 05/10] feat: add dns-resolver-timeout property defaulting to 5s Adds a new `dns-resolver-timeout` global nginx property that controls how long nginx waits on the configured DNS resolver before giving up and returning 502. Without an explicit timeout nginx blocks on the resolver for its built-in 30s default, which means clients see hung requests instead of a fast 502 when coredns-docker is unreachable. The 5s default keeps failure-mode requests short while leaving headroom over a healthy local resolver round trip. The bundled `nginx.conf.sigil` emits `resolver_timeout {{ $.NGINX_DNS_RESOLVER_TIMEOUT }}` next to the existing `resolver` directive, and the outage-mode bats and docker-deploy tests now expect the curl call to complete inside that window. --- docs/appendices/0.39.0-migration-guide.md | 7 ++++--- .../file-formats/nginx-conf-sigil.md | 5 +++-- docs/networking/proxies/nginx.md | 3 ++- plugins/nginx-vhosts/functions | 2 ++ plugins/nginx-vhosts/internal-functions | 21 +++++++++++++++++++ plugins/nginx-vhosts/nginx_vhosts.go | 17 +++++++++++++++ plugins/nginx-vhosts/report.go | 1 + .../src/nginx-property/nginx-property.go | 6 ++++++ plugins/nginx-vhosts/subcommands/set | 6 +++--- plugins/nginx-vhosts/template_test.go | 10 +++++++++ .../nginx-vhosts/templates/nginx.conf.sigil | 2 ++ tests/test_coredns_outage | 2 +- tests/unit/nginx-vhosts_17.bats | 2 +- 13 files changed, 73 insertions(+), 11 deletions(-) diff --git a/docs/appendices/0.39.0-migration-guide.md b/docs/appendices/0.39.0-migration-guide.md index 4b40ebe05..616ddf5f1 100644 --- a/docs/appendices/0.39.0-migration-guide.md +++ b/docs/appendices/0.39.0-migration-guide.md @@ -9,12 +9,13 @@ - Changing a builder's `stack` now clears that builder's build cache so the next deploy starts fresh. For herokuish this removes the `cache-` Docker volume; for pack the next `pack build` runs with `--clear-cache`. - The `dokku` Debian package now declares `coredns-docker (>= 0.6.0)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. - The bundled `nginx.conf.sigil` now uses an nginx `resolver` directive and a variable-based `proxy_pass` against `..docker` instead of pinning the upstream to a literal IP. This is what addresses the long-standing post-reboot and post-`apt-upgrade-docker` stale-vhost bug ([#8313](https://github.com/dokku/dokku/issues/8313)) - the rendered upstream re-resolves at request time via coredns-docker, so a host reboot or single-container restart no longer leaves nginx routing to a stale IP. See the [nginx documentation](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for details. -- Two new global nginx properties control the resolver path: +- Three new global nginx properties control the resolver path: - `dns-resolver` (default: `127.0.0.1:1053`) - the address nginx talks to. + - `dns-resolver-timeout` (default: `5s`) - how long nginx waits for a DNS response before giving up and returning 502. The short default keeps clients from waiting on nginx's built-in 30s timeout when coredns-docker is unreachable. - `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. - Both can be set via `dokku nginx:set --global `. + All three can be set via `dokku nginx:set --global `. - The literal string `off` is the sentinel for disabling the resolver path. **Setting `dns-resolver` to the empty string does not disable it** - that deletes the property file and the new default (`127.0.0.1:1053`) re-engages. To revert to the legacy IP-based upstream block on hosts where coredns-docker cannot run (source/git installs, locally held coredns-docker package, postinst failures): ```shell @@ -22,7 +23,7 @@ dokku proxy:build-config --all ``` -- Vendored `nginx.conf.sigil` templates are unaffected by the change. The existing `DOKKU_APP__LISTENERS` variable continues to be populated with the IP-based listener list exactly as before, so templates that only reference it continue to render IP-based upstreams regardless of the new property defaults. To opt a vendored template into the resolver path, switch its `proxy_pass` line to use the new `DOKKU_APP__LISTENER_HOST`, `NGINX_DNS_RESOLVER`, and `NGINX_DNS_ZONE` sigil variables - see the [nginx.conf.sigil reference](/docs/appendices/file-formats/nginx-conf-sigil.md#upstream-rendering-ip-based-vs-dns-based) for the exact snippet. +- Vendored `nginx.conf.sigil` templates are unaffected by the change. The existing `DOKKU_APP__LISTENERS` variable continues to be populated with the IP-based listener list exactly as before, so templates that only reference it continue to render IP-based upstreams regardless of the new property defaults. To opt a vendored template into the resolver path, switch its `proxy_pass` line to use the new `DOKKU_APP__LISTENER_HOST`, `NGINX_DNS_RESOLVER`, `NGINX_DNS_RESOLVER_TIMEOUT`, and `NGINX_DNS_ZONE` sigil variables - see the [nginx.conf.sigil reference](/docs/appendices/file-formats/nginx-conf-sigil.md#upstream-rendering-ip-based-vs-dns-based) for the exact snippet. ### Stack property migrated to builder plugins diff --git a/docs/appendices/file-formats/nginx-conf-sigil.md b/docs/appendices/file-formats/nginx-conf-sigil.md index bb9a063f6..2d7ef8b6e 100644 --- a/docs/appendices/file-formats/nginx-conf-sigil.md +++ b/docs/appendices/file-formats/nginx-conf-sigil.md @@ -96,13 +96,14 @@ This path renders the IP that Dokku saw when the config was last built. After a ### DNS-based upstream (resolver path) -The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.6.0`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. +The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_RESOLVER_TIMEOUT` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.6.0`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. The bundled template emits this snippet: ```nginx {{ if ne $.NGINX_DNS_RESOLVER "off" }} resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; +resolver_timeout {{ $.NGINX_DNS_RESOLVER_TIMEOUT }}; set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; proxy_pass http://$dokku_upstream; {{ else }} @@ -112,7 +113,7 @@ proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; The `$dokku_upstream` variable name is intentionally namespaced (rather than `$upstream`) so vendored templates that already define `$upstream` will not collide if they adopt the resolver branch. -`NGINX_DNS_RESOLVER` defaults to `127.0.0.1:1053` and `NGINX_DNS_ZONE` defaults to `docker`; both are configurable via `dokku nginx:set --global dns-resolver ...` and `dokku nginx:set --global dns-zone ...`. The literal string `off` is the sentinel for disabling the resolver branch in the bundled template - empty strings are treated as "use the default" because the property is stored on disk and an empty value deletes the property file. +`NGINX_DNS_RESOLVER` defaults to `127.0.0.1:1053`, `NGINX_DNS_RESOLVER_TIMEOUT` defaults to `5s`, and `NGINX_DNS_ZONE` defaults to `docker`; all three are configurable via `dokku nginx:set --global dns-resolver ...`, `dokku nginx:set --global dns-resolver-timeout ...`, and `dokku nginx:set --global dns-zone ...`. The literal string `off` is the sentinel for disabling the resolver branch in the bundled template - empty strings are treated as "use the default" because the property is stored on disk and an empty value deletes the property file. The short default `dns-resolver-timeout` keeps clients from waiting on nginx's built-in 30s timeout when coredns-docker is unreachable; nginx returns 502 after the configured interval instead. ### Choosing between the two diff --git a/docs/networking/proxies/nginx.md b/docs/networking/proxies/nginx.md index 8f5e0ab63..ede8157ae 100644 --- a/docs/networking/proxies/nginx.md +++ b/docs/networking/proxies/nginx.md @@ -39,9 +39,10 @@ Dokku resolves nginx upstreams through [coredns-docker](https://github.com/dokku No `systemd-resolved` or `/etc/resolv.conf` change is required - nginx talks to `coredns-docker` directly via the `resolver` directive in each app's `nginx.conf`. -The behavior is controlled by two global nginx properties: +The behavior is controlled by three global nginx properties: - `dns-resolver` (default: `127.0.0.1:1053`) - the resolver address nginx talks to. +- `dns-resolver-timeout` (default: `5s`) - how long nginx waits for a DNS response before giving up and returning 502. The default is short so requests fail fast when coredns-docker is unreachable rather than hanging on the nginx built-in default (30s). - `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. To revert to the legacy IP-based upstream block (for instance on a host where `coredns-docker` cannot be installed): diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index 051a8f525..747f45bdd 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -325,6 +325,7 @@ fn-nginx-vhosts-render-template() { local NGINX_BIND_ADDRESS_IP4="$(fn-nginx-computed-bind-address-ipv4 "$APP")" local NGINX_BIND_ADDRESS_IP6="$(fn-nginx-computed-bind-address-ipv6 "$APP")" local NGINX_DNS_RESOLVER="$(fn-nginx-computed-dns-resolver "$APP")" + local NGINX_DNS_RESOLVER_TIMEOUT="$(fn-nginx-computed-dns-resolver-timeout "$APP")" local NGINX_DNS_ZONE="$(fn-nginx-computed-dns-zone "$APP")" local NGINX_UNDERSCORE_IN_HEADERS="$(fn-nginx-computed-underscore-in-headers "$APP")" local PROXY_X_FORWARDED_FOR="$(fn-nginx-computed-x-forwarded-for-value "$APP")" @@ -357,6 +358,7 @@ fn-nginx-vhosts-render-template() { NGINX_BIND_ADDRESS_IP4="$NGINX_BIND_ADDRESS_IP4" NGINX_BIND_ADDRESS_IP6="$NGINX_BIND_ADDRESS_IP6" NGINX_DNS_RESOLVER="$NGINX_DNS_RESOLVER" + NGINX_DNS_RESOLVER_TIMEOUT="$NGINX_DNS_RESOLVER_TIMEOUT" NGINX_DNS_ZONE="$NGINX_DNS_ZONE" HTTP2_DIRECTIVE_SUPPORTED="$HTTP2_DIRECTIVE_SUPPORTED" HTTP2_PUSH_SUPPORTED="$HTTP2_PUSH_SUPPORTED" diff --git a/plugins/nginx-vhosts/internal-functions b/plugins/nginx-vhosts/internal-functions index 3b9e378eb..63f63aa35 100755 --- a/plugins/nginx-vhosts/internal-functions +++ b/plugins/nginx-vhosts/internal-functions @@ -193,6 +193,27 @@ fn-nginx-global-dns-resolver() { fn-get-property --app "$APP" --global "dns-resolver" } +fn-nginx-dns-resolver-timeout() { + declare desc="get the configured dns-resolver-timeout value" + declare APP="$1" + + fn-get-property --app "$APP" "dns-resolver-timeout" +} + +fn-nginx-computed-dns-resolver-timeout() { + declare desc="get the computed dns-resolver-timeout value" + declare APP="$1" + + fn-get-property --app "$APP" --computed "dns-resolver-timeout" +} + +fn-nginx-global-dns-resolver-timeout() { + declare desc="get the global dns-resolver-timeout value" + declare APP="$1" + + fn-get-property --app "$APP" --global "dns-resolver-timeout" +} + fn-nginx-dns-zone() { declare desc="get the configured dns-zone value" declare APP="$1" diff --git a/plugins/nginx-vhosts/nginx_vhosts.go b/plugins/nginx-vhosts/nginx_vhosts.go index bd6b14cae..7d562e350 100644 --- a/plugins/nginx-vhosts/nginx_vhosts.go +++ b/plugins/nginx-vhosts/nginx_vhosts.go @@ -167,6 +167,23 @@ func GlobalDnsResolver() string { return common.PropertyGetDefault("nginx", "--global", "dns-resolver", "127.0.0.1:1053") } +func AppDnsResolverTimeout(appName string) string { + return common.PropertyGet("nginx", appName, "dns-resolver-timeout") +} + +func ComputedDnsResolverTimeout(appName string) string { + appValue := AppDnsResolverTimeout(appName) + if appValue != "" { + return appValue + } + + return GlobalDnsResolverTimeout() +} + +func GlobalDnsResolverTimeout() string { + return common.PropertyGetDefault("nginx", "--global", "dns-resolver-timeout", "5s") +} + func AppDnsZone(appName string) string { return common.PropertyGet("nginx", appName, "dns-zone") } diff --git a/plugins/nginx-vhosts/report.go b/plugins/nginx-vhosts/report.go index f6d29bd68..327d819e8 100644 --- a/plugins/nginx-vhosts/report.go +++ b/plugins/nginx-vhosts/report.go @@ -30,6 +30,7 @@ func nginxReportProps() []nginxReportProp { {"client-max-body-size", AppClientMaxBodySize, ComputedClientMaxBodySize, wrapGlobal(GlobalClientMaxBodySize)}, {"disable-custom-config", AppDisableCustomConfig, ComputedDisableCustomConfig, wrapGlobal(GlobalDisableCustomConfig)}, {"dns-resolver", AppDnsResolver, ComputedDnsResolver, wrapGlobal(GlobalDnsResolver)}, + {"dns-resolver-timeout", AppDnsResolverTimeout, ComputedDnsResolverTimeout, wrapGlobal(GlobalDnsResolverTimeout)}, {"dns-zone", AppDnsZone, ComputedDnsZone, wrapGlobal(GlobalDnsZone)}, {"error-log-path", AppErrorLogPath, ComputedErrorLogPath, wrapGlobal(GlobalErrorLogPath)}, {"hsts-include-subdomains", AppHSTSIncludeSubdomains, ComputedHSTSIncludeSubdomains, wrapGlobal(GlobalHSTSIncludeSubdomains)}, diff --git a/plugins/nginx-vhosts/src/nginx-property/nginx-property.go b/plugins/nginx-vhosts/src/nginx-property/nginx-property.go index bd1e3c28b..167185ec5 100644 --- a/plugins/nginx-vhosts/src/nginx-property/nginx-property.go +++ b/plugins/nginx-vhosts/src/nginx-property/nginx-property.go @@ -53,6 +53,8 @@ func appValue(appName string, property string) string { value = nginx_vhosts.AppDisableCustomConfig(appName) case "dns-resolver": value = nginx_vhosts.AppDnsResolver(appName) + case "dns-resolver-timeout": + value = nginx_vhosts.AppDnsResolverTimeout(appName) case "dns-zone": value = nginx_vhosts.AppDnsZone(appName) case "error-log-path": @@ -129,6 +131,8 @@ func computedValue(appName string, property string) string { value = nginx_vhosts.ComputedDisableCustomConfig(appName) case "dns-resolver": value = nginx_vhosts.ComputedDnsResolver(appName) + case "dns-resolver-timeout": + value = nginx_vhosts.ComputedDnsResolverTimeout(appName) case "dns-zone": value = nginx_vhosts.ComputedDnsZone(appName) case "error-log-path": @@ -205,6 +209,8 @@ func globalValue(appName string, property string) string { value = nginx_vhosts.GlobalDisableCustomConfig() case "dns-resolver": value = nginx_vhosts.GlobalDnsResolver() + case "dns-resolver-timeout": + value = nginx_vhosts.GlobalDnsResolverTimeout() case "dns-zone": value = nginx_vhosts.GlobalDnsZone() case "error-log-path": diff --git a/plugins/nginx-vhosts/subcommands/set b/plugins/nginx-vhosts/subcommands/set index 439d421d4..4685bc030 100755 --- a/plugins/nginx-vhosts/subcommands/set +++ b/plugins/nginx-vhosts/subcommands/set @@ -9,13 +9,13 @@ cmd-nginx-set() { declare cmd="nginx:set" [[ "$1" == "$cmd" ]] && shift 1 declare APP="$1" KEY="$2" VALUE="$3" - local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") - local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") + local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-resolver-timeout" "dns-zone" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") + local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-resolver-timeout" "dns-zone" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") [[ -z "$KEY" ]] && dokku_log_fail "No key specified" if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then - dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, dns-resolver, dns-zone, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl" + dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, dns-resolver, dns-resolver-timeout, dns-zone, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl" fi if ! fn-in-array "$KEY" "${GLOBAL_KEYS[@]}" && [[ "$APP" == "--global" ]]; then diff --git a/plugins/nginx-vhosts/template_test.go b/plugins/nginx-vhosts/template_test.go index ebfb02cea..0d95c27e6 100644 --- a/plugins/nginx-vhosts/template_test.go +++ b/plugins/nginx-vhosts/template_test.go @@ -39,6 +39,7 @@ func defaultVars() map[string]interface{} { "NGINX_BIND_ADDRESS_IP4": "", "NGINX_BIND_ADDRESS_IP6": "::", "NGINX_DNS_RESOLVER": "127.0.0.1:1053", + "NGINX_DNS_RESOLVER_TIMEOUT": "5s", "NGINX_DNS_ZONE": "docker", "NGINX_ACCESS_LOG_PATH": "/var/log/nginx/app-access.log", "NGINX_ACCESS_LOG_FORMAT": "", @@ -99,6 +100,7 @@ func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { out := renderTemplate(t, defaultVars()) mustContain(t, out, "listen [::]:80;") mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;") + mustContain(t, out, "resolver_timeout 5s;") mustContain(t, out, `set $dokku_upstream "app.web.docker:5000";`) mustContain(t, out, "proxy_pass http://$dokku_upstream;") mustNotContain(t, out, "upstream app-5000 {") @@ -109,6 +111,14 @@ func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { mustNotContain(t, out, "http2_push_preload") } +func TestTemplate_ResolverTimeoutOverride(t *testing.T) { + v := defaultVars() + v["NGINX_DNS_RESOLVER_TIMEOUT"] = "2s" + out := renderTemplate(t, v) + mustContain(t, out, "resolver_timeout 2s;") + mustNotContain(t, out, "resolver_timeout 5s;") +} + func TestTemplate_HTTPOnlyBasicProxyWithResolverOff(t *testing.T) { v := defaultVars() v["NGINX_DNS_RESOLVER"] = "off" diff --git a/plugins/nginx-vhosts/templates/nginx.conf.sigil b/plugins/nginx-vhosts/templates/nginx.conf.sigil index 57e6ec3c4..5bb22703b 100644 --- a/plugins/nginx-vhosts/templates/nginx.conf.sigil +++ b/plugins/nginx-vhosts/templates/nginx.conf.sigil @@ -86,6 +86,7 @@ server { {{ if ne $.NGINX_DNS_RESOLVER "off" }} resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; + resolver_timeout {{ $.NGINX_DNS_RESOLVER_TIMEOUT }}; set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; proxy_pass http://$dokku_upstream; {{ else }} @@ -174,6 +175,7 @@ server { location / { {{ if ne $.NGINX_DNS_RESOLVER "off" }} resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; + resolver_timeout {{ $.NGINX_DNS_RESOLVER_TIMEOUT }}; set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; grpc_pass grpc://$dokku_upstream; {{ else }} diff --git a/tests/test_coredns_outage b/tests/test_coredns_outage index f1a168b01..4b4ebc0f7 100755 --- a/tests/test_coredns_outage +++ b/tests/test_coredns_outage @@ -47,7 +47,7 @@ start_coredns_docker() { } curl_status() { - curl -kSso /dev/null -w "%{http_code}" -m 5 "http://$APP.$TARGET/" + curl -kSso /dev/null -w "%{http_code}" -m 10 "http://$APP.$TARGET/" } wait_for_status() { diff --git a/tests/unit/nginx-vhosts_17.bats b/tests/unit/nginx-vhosts_17.bats index 6f51c6356..2e9fe0865 100644 --- a/tests/unit/nginx-vhosts_17.bats +++ b/tests/unit/nginx-vhosts_17.bats @@ -150,7 +150,7 @@ teardown() { echo "status: $status" assert_success - run curl --connect-to "$TEST_APP.$DOKKU_DOMAIN:80:localhost:80" -kSso /dev/null -w "%{http_code}" -m 5 "http://$TEST_APP.$DOKKU_DOMAIN/" + run curl --connect-to "$TEST_APP.$DOKKU_DOMAIN:80:localhost:80" -kSso /dev/null -w "%{http_code}" -m 10 "http://$TEST_APP.$DOKKU_DOMAIN/" echo "output: $output" echo "status: $status" assert_output "502" From bf2795640c4a6326baaa68448f7b21c6b231573a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 6 Nov 2026 20:51:33 -0500 Subject: [PATCH 06/10] feat: make coredns-docker upstream resolution opt-in The bundled `nginx.conf.sigil` keeps rendering IP-based upstreams unless `dns-resolver` is set, for example with `dokku nginx:set --global dns-resolver 127.0.0.1:1053`. Unsetting the property reverts to IP-based upstreams, replacing the previous `off` sentinel. --- debian/control | 2 +- docs/appendices/0.39.0-migration-guide.md | 22 +++---- .../file-formats/nginx-conf-sigil.md | 6 +- docs/getting-started/install/docker.md | 2 +- docs/networking/proxies/nginx.md | 30 ++++++--- plugins/nginx-vhosts/nginx_vhosts.go | 2 +- plugins/nginx-vhosts/template_test.go | 45 +++++++------ .../nginx-vhosts/templates/nginx.conf.sigil | 6 +- tests/test_coredns_outage | 2 + tests/unit/nginx-vhosts_17.bats | 63 +++++++++++-------- 10 files changed, 107 insertions(+), 73 deletions(-) diff --git a/debian/control b/debian/control index 4d5e30a93..7060a84c1 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.38.34 Section: web Priority: optional Architecture: amd64 -Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, coredns-docker, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux +Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, coredns-docker (>= 0.6.0), docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux Recommends: herokuish, bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, jq, nginx (>= 1.8.0) | openresty, bind9-dnsutils, debconf, plugn, sudo, python3, rsyslog Maintainer: Jose Diaz-Gonzalez diff --git a/docs/appendices/0.39.0-migration-guide.md b/docs/appendices/0.39.0-migration-guide.md index 616ddf5f1..c685b6ac0 100644 --- a/docs/appendices/0.39.0-migration-guide.md +++ b/docs/appendices/0.39.0-migration-guide.md @@ -8,22 +8,22 @@ - The internal `buildpack-stack-name` plugin trigger has been removed. Each builder now reads its own `stack` property directly during the build. - Changing a builder's `stack` now clears that builder's build cache so the next deploy starts fresh. For herokuish this removes the `cache-` Docker volume; for pack the next `pack build` runs with `--clear-cache`. - The `dokku` Debian package now declares `coredns-docker (>= 0.6.0)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. -- The bundled `nginx.conf.sigil` now uses an nginx `resolver` directive and a variable-based `proxy_pass` against `..docker` instead of pinning the upstream to a literal IP. This is what addresses the long-standing post-reboot and post-`apt-upgrade-docker` stale-vhost bug ([#8313](https://github.com/dokku/dokku/issues/8313)) - the rendered upstream re-resolves at request time via coredns-docker, so a host reboot or single-container restart no longer leaves nginx routing to a stale IP. See the [nginx documentation](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for details. -- Three new global nginx properties control the resolver path: - - - `dns-resolver` (default: `127.0.0.1:1053`) - the address nginx talks to. - - `dns-resolver-timeout` (default: `5s`) - how long nginx waits for a DNS response before giving up and returning 502. The short default keeps clients from waiting on nginx's built-in 30s timeout when coredns-docker is unreachable. - - `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. - - All three can be set via `dokku nginx:set --global `. -- The literal string `off` is the sentinel for disabling the resolver path. **Setting `dns-resolver` to the empty string does not disable it** - that deletes the property file and the new default (`127.0.0.1:1053`) re-engages. To revert to the legacy IP-based upstream block on hosts where coredns-docker cannot run (source/git installs, locally held coredns-docker package, postinst failures): +- The bundled `nginx.conf.sigil` can now use an nginx `resolver` directive and a variable-based `proxy_pass` against `..docker` instead of pinning the upstream to a literal IP. This addresses the long-standing post-reboot and post-`apt-upgrade-docker` stale-vhost bug ([#8313](https://github.com/dokku/dokku/issues/8313)) - the rendered upstream re-resolves at request time via coredns-docker, so a host reboot or single-container restart no longer leaves nginx routing to a stale IP. This behavior is opt-in, and upstreams remain IP-based unless `dns-resolver` is set: ```shell - dokku nginx:set --global dns-resolver off + dokku nginx:set --global dns-resolver 127.0.0.1:1053 dokku proxy:build-config --all ``` -- Vendored `nginx.conf.sigil` templates are unaffected by the change. The existing `DOKKU_APP__LISTENERS` variable continues to be populated with the IP-based listener list exactly as before, so templates that only reference it continue to render IP-based upstreams regardless of the new property defaults. To opt a vendored template into the resolver path, switch its `proxy_pass` line to use the new `DOKKU_APP__LISTENER_HOST`, `NGINX_DNS_RESOLVER`, `NGINX_DNS_RESOLVER_TIMEOUT`, and `NGINX_DNS_ZONE` sigil variables - see the [nginx.conf.sigil reference](/docs/appendices/file-formats/nginx-conf-sigil.md#upstream-rendering-ip-based-vs-dns-based) for the exact snippet. + See the [nginx documentation](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for details. +- Three new nginx properties control the resolver path: + + - `dns-resolver` (default: none) - the address nginx talks to. Resolver-based upstreams are only rendered when this is set. + - `dns-resolver-timeout` (default: `5s`) - how long nginx waits for a DNS response before giving up and returning 502. The short default keeps clients from waiting on nginx's built-in 30s timeout when coredns-docker is unreachable. + - `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. + + All three can be set per app or globally via `dokku nginx:set`. +- Vendored `nginx.conf.sigil` templates are unaffected by the change. The existing `DOKKU_APP__LISTENERS` variable continues to be populated with the IP-based listener list exactly as before, so templates that only reference it continue to render IP-based upstreams regardless of the new properties. To opt a vendored template into the resolver path, switch its `proxy_pass` line to use the new `DOKKU_APP__LISTENER_HOST`, `NGINX_DNS_RESOLVER`, `NGINX_DNS_RESOLVER_TIMEOUT`, and `NGINX_DNS_ZONE` sigil variables - see the [nginx.conf.sigil reference](/docs/appendices/file-formats/nginx-conf-sigil.md#upstream-rendering-ip-based-vs-dns-based) for the exact snippet. ### Stack property migrated to builder plugins diff --git a/docs/appendices/file-formats/nginx-conf-sigil.md b/docs/appendices/file-formats/nginx-conf-sigil.md index 2d7ef8b6e..c0324cacf 100644 --- a/docs/appendices/file-formats/nginx-conf-sigil.md +++ b/docs/appendices/file-formats/nginx-conf-sigil.md @@ -101,7 +101,7 @@ The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NG The bundled template emits this snippet: ```nginx -{{ if ne $.NGINX_DNS_RESOLVER "off" }} +{{ if $.NGINX_DNS_RESOLVER }} resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; resolver_timeout {{ $.NGINX_DNS_RESOLVER_TIMEOUT }}; set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; @@ -113,8 +113,8 @@ proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; The `$dokku_upstream` variable name is intentionally namespaced (rather than `$upstream`) so vendored templates that already define `$upstream` will not collide if they adopt the resolver branch. -`NGINX_DNS_RESOLVER` defaults to `127.0.0.1:1053`, `NGINX_DNS_RESOLVER_TIMEOUT` defaults to `5s`, and `NGINX_DNS_ZONE` defaults to `docker`; all three are configurable via `dokku nginx:set --global dns-resolver ...`, `dokku nginx:set --global dns-resolver-timeout ...`, and `dokku nginx:set --global dns-zone ...`. The literal string `off` is the sentinel for disabling the resolver branch in the bundled template - empty strings are treated as "use the default" because the property is stored on disk and an empty value deletes the property file. The short default `dns-resolver-timeout` keeps clients from waiting on nginx's built-in 30s timeout when coredns-docker is unreachable; nginx returns 502 after the configured interval instead. +`NGINX_DNS_RESOLVER` is empty by default, `NGINX_DNS_RESOLVER_TIMEOUT` defaults to `5s`, and `NGINX_DNS_ZONE` defaults to `docker`; all three are configurable via `dokku nginx:set --global dns-resolver ...`, `dokku nginx:set --global dns-resolver-timeout ...`, and `dokku nginx:set --global dns-zone ...`. The bundled template only renders the resolver branch when `NGINX_DNS_RESOLVER` is set, for example to `127.0.0.1:1053`. Unsetting `dns-resolver` switches the bundled template back to the IP-based upstream. The short default `dns-resolver-timeout` keeps clients from waiting on nginx's built-in 30s timeout when coredns-docker is unreachable; nginx returns 502 after the configured interval instead. ### Choosing between the two -The DNS-based path is the default for the bundled template. Vendored templates that have not been updated continue to use `DOKKU_APP__LISTENERS` and are unaffected by the global default. The DNS-based variables are populated regardless of whether `dns-resolver` is `off` or not, so a vendored template can adopt the resolver path on its own schedule by switching its rendering snippet, even if other apps on the host are still using IP-based upstreams. +The IP-based path is the default for the bundled template, and the DNS-based path is used once `dns-resolver` is set. Vendored templates that have not been updated continue to use `DOKKU_APP__LISTENERS` and are unaffected by the `dns-resolver` setting. The `DOKKU_APP__LISTENER_HOST` and `NGINX_DNS_ZONE` variables are populated regardless of whether `dns-resolver` is set, so a vendored template can adopt the resolver path on its own schedule by switching its rendering snippet, even if other apps on the host are still using IP-based upstreams. diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md index 15d7efbe7..e79a841d7 100644 --- a/docs/getting-started/install/docker.md +++ b/docs/getting-started/install/docker.md @@ -88,7 +88,7 @@ Dokku is run in the following configuration: - Image build cache is set to the data dir + `/home/dokku`. - The docker socket is mounted into container. - `DOKKU_HOST_ROOT` and `DOKKU_LIB_HOST_ROOT` tell Dokku how the host Docker daemon sees those same trees when creating bind mounts (app repos and plugin data such as named storage). -- `coredns-docker` (a CoreDNS plugin that serves DNS records for sibling Docker containers) is installed transitively via the dokku deb's `Depends:`, runs under runit on `127.0.0.1:1053` inside the container, and is what the `nginx-vhosts` `dns-resolver` default points at. This is what makes the `..docker` upstream resolution work out of the box - see the [nginx proxy doc](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for the operator-facing behavior. +- `coredns-docker` (a CoreDNS plugin that serves DNS records for sibling Docker containers) is installed transitively via the dokku deb's `Depends:`, runs under runit on `127.0.0.1:1053` inside the container, and is what the `nginx-vhosts` `dns-resolver` property can point at to enable `..docker` upstream resolution - see the [nginx proxy doc](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for the operator-facing behavior. Application repositories, plugin config, as well as plugin data are persisted to disk within the specified host directory for `/var/lib/dokku`. `DOKKU_LIB_HOST_ROOT` must point at that host path's `var/lib/dokku` subtree so Docker `-v` mounts resolve under the data volume rather than under the host's own `/var/lib/dokku`. diff --git a/docs/networking/proxies/nginx.md b/docs/networking/proxies/nginx.md index ede8157ae..1e07b6758 100644 --- a/docs/networking/proxies/nginx.md +++ b/docs/networking/proxies/nginx.md @@ -33,27 +33,34 @@ Nginx will proxy the requests in a [round-robin balancing fashion](http://nginx. > [!IMPORTANT] > New as of 0.39.0 -Dokku resolves nginx upstreams through [coredns-docker](https://github.com/dokku/coredns-docker), a CoreDNS plugin that watches the Docker daemon and serves fresh `A` records keyed off container labels. The nginx `resolver` directive points at coredns-docker's local socket (`127.0.0.1:1053`) and the rendered upstream is a variable-based `proxy_pass`, which forces nginx to re-resolve at request time honoring DNS TTL. This is how nginx avoids serving the wrong vhost after a host reboot, after `apt upgrade docker`, or after a single container restarts mid-runtime: the on-disk nginx config no longer pins a stale IP. +Dokku can resolve nginx upstreams through [coredns-docker](https://github.com/dokku/coredns-docker), a CoreDNS plugin that watches the Docker daemon and serves fresh `A` records keyed off container labels. When enabled, the nginx `resolver` directive points at coredns-docker's local socket (`127.0.0.1:1053`) and the rendered upstream is a variable-based `proxy_pass`, which forces nginx to re-resolve at request time honoring DNS TTL. This is how nginx avoids serving the wrong vhost after a host reboot, after `apt upgrade docker`, or after a single container restarts mid-runtime: the on-disk nginx config no longer pins a stale IP. This behavior is opt-in, and nginx uses IP-based `upstream` blocks by default. `coredns-docker` (>= `0.6.0`) is installed automatically as a transitive `Depends:` of the `dokku` Debian package, so `apt install dokku` and `apt upgrade dokku` pull it in without operator action. The shipped `/etc/coredns/Corefile` already binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from Dokku's existing `com.dokku.app-name` and `com.dokku.process-type` container labels. Multiple containers for the same process type collapse onto one multi-A record set, so scaled apps load-balance across instances via DNS without any per-container configuration. The bundled Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. No `systemd-resolved` or `/etc/resolv.conf` change is required - nginx talks to `coredns-docker` directly via the `resolver` directive in each app's `nginx.conf`. -The behavior is controlled by three global nginx properties: - -- `dns-resolver` (default: `127.0.0.1:1053`) - the resolver address nginx talks to. -- `dns-resolver-timeout` (default: `5s`) - how long nginx waits for a DNS response before giving up and returning 502. The default is short so requests fail fast when coredns-docker is unreachable rather than hanging on the nginx built-in default (30s). -- `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. - -To revert to the legacy IP-based upstream block (for instance on a host where `coredns-docker` cannot be installed): +To enable resolver-based upstreams: ```shell -dokku nginx:set --global dns-resolver off +dokku nginx:set --global dns-resolver 127.0.0.1:1053 dokku proxy:build-config --all ``` +To revert to the IP-based upstream block, unset the property and rebuild the config: + +```shell +dokku nginx:set --global dns-resolver +dokku proxy:build-config --all +``` + +The behavior is controlled by three nginx properties, each settable per app or globally: + +- `dns-resolver` (default: none) - the resolver address nginx talks to. Resolver-based upstreams are only rendered when this is set. +- `dns-resolver-timeout` (default: `5s`) - how long nginx waits for a DNS response before giving up and returning 502. The default is short so requests fail fast when coredns-docker is unreachable rather than hanging on the nginx built-in default (30s). +- `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. + > [!NOTE] -> The sentinel for disabling the feature is the literal string `off`. Setting `dns-resolver` to the empty string (`""`) does **not** disable it - that deletes the property and the default (`127.0.0.1:1053`) re-engages. Use `off` to disable, the default to re-enable. +> When `dns-resolver` is set, no `upstream` block is rendered for the `web` process, so `proxy-keepalive` has no effect on it. Upstreams for [proxy routes](/docs/networking/proxy-management.md#path-based-routing-to-non-web-processes) to other process types remain IP-based. See [the `nginx.conf.sigil` reference](/docs/appendices/file-formats/nginx-conf-sigil.md) for the sigil variables that drive the resolver-based render, and how vendored templates can adopt or skip it. @@ -672,6 +679,9 @@ All nginx-vhosts properties are settable at both the app and global scope. | `client-header-timeout` | app + global | `60s` | `--nginx-client-header-timeout`, `--nginx-global-client-header-timeout`, `--nginx-computed-client-header-timeout` | Time allowed to read the request header from the client | | `client-max-body-size` | app + global | `1m` | `--nginx-client-max-body-size`, `--nginx-global-client-max-body-size`, `--nginx-computed-client-max-body-size` | Maximum allowed request body size | | `disable-custom-config` | app + global | `false` | `--nginx-disable-custom-config`, `--nginx-global-disable-custom-config`, `--nginx-computed-disable-custom-config` | When `true`, ignores app-supplied `nginx.conf.d/*.conf` snippets | +| `dns-resolver` | app + global | none | `--nginx-dns-resolver`, `--nginx-global-dns-resolver`, `--nginx-computed-dns-resolver` | Resolver address used for DNS-based upstreams. When unset, upstreams are IP-based | +| `dns-resolver-timeout` | app + global | `5s` | `--nginx-dns-resolver-timeout`, `--nginx-global-dns-resolver-timeout`, `--nginx-computed-dns-resolver-timeout` | Time nginx waits for a DNS response from the resolver | +| `dns-zone` | app + global | `docker` | `--nginx-dns-zone`, `--nginx-global-dns-zone`, `--nginx-computed-dns-zone` | Zone appended to `.` for DNS-based upstream hostnames | | `error-log-path` | app + global | _`{nginx-log-root}/{app}-error.log`_ | `--nginx-error-log-path`, `--nginx-global-error-log-path`, `--nginx-computed-error-log-path` | Path inside the nginx container where error logs are written | | `hsts` | app + global | `true` | `--nginx-hsts`, `--nginx-global-hsts`, `--nginx-computed-hsts` | When `true`, emits a `Strict-Transport-Security` header on HTTPS responses | | `hsts-include-subdomains` | app + global | `true` | `--nginx-hsts-include-subdomains`, `--nginx-global-hsts-include-subdomains`, `--nginx-computed-hsts-include-subdomains` | Adds the `includeSubDomains` directive to the HSTS header | diff --git a/plugins/nginx-vhosts/nginx_vhosts.go b/plugins/nginx-vhosts/nginx_vhosts.go index 7d562e350..7a78c84f1 100644 --- a/plugins/nginx-vhosts/nginx_vhosts.go +++ b/plugins/nginx-vhosts/nginx_vhosts.go @@ -164,7 +164,7 @@ func ComputedDnsResolver(appName string) string { } func GlobalDnsResolver() string { - return common.PropertyGetDefault("nginx", "--global", "dns-resolver", "127.0.0.1:1053") + return common.PropertyGet("nginx", "--global", "dns-resolver") } func AppDnsResolverTimeout(appName string) string { diff --git a/plugins/nginx-vhosts/template_test.go b/plugins/nginx-vhosts/template_test.go index 0d95c27e6..7d9927743 100644 --- a/plugins/nginx-vhosts/template_test.go +++ b/plugins/nginx-vhosts/template_test.go @@ -38,7 +38,7 @@ func defaultVars() map[string]interface{} { "PROXY_KEEPALIVE": "", "NGINX_BIND_ADDRESS_IP4": "", "NGINX_BIND_ADDRESS_IP6": "::", - "NGINX_DNS_RESOLVER": "127.0.0.1:1053", + "NGINX_DNS_RESOLVER": "", "NGINX_DNS_RESOLVER_TIMEOUT": "5s", "NGINX_DNS_ZONE": "docker", "NGINX_ACCESS_LOG_PATH": "/var/log/nginx/app-access.log", @@ -99,11 +99,10 @@ func mustNotContain(t *testing.T, out, needle string) { func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { out := renderTemplate(t, defaultVars()) mustContain(t, out, "listen [::]:80;") - mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;") - mustContain(t, out, "resolver_timeout 5s;") - mustContain(t, out, `set $dokku_upstream "app.web.docker:5000";`) - mustContain(t, out, "proxy_pass http://$dokku_upstream;") - mustNotContain(t, out, "upstream app-5000 {") + mustContain(t, out, "proxy_pass http://app-5000;") + mustContain(t, out, "upstream app-5000 {") + mustNotContain(t, out, "resolver ") + mustNotContain(t, out, "$dokku_upstream") mustContain(t, out, "error_page 500 501 502 503") mustContain(t, out, "server_name app.example.com;") mustNotContain(t, out, "ssl_certificate") @@ -113,21 +112,23 @@ func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { func TestTemplate_ResolverTimeoutOverride(t *testing.T) { v := defaultVars() + v["NGINX_DNS_RESOLVER"] = "127.0.0.1:1053" v["NGINX_DNS_RESOLVER_TIMEOUT"] = "2s" out := renderTemplate(t, v) mustContain(t, out, "resolver_timeout 2s;") mustNotContain(t, out, "resolver_timeout 5s;") } -func TestTemplate_HTTPOnlyBasicProxyWithResolverOff(t *testing.T) { +func TestTemplate_HTTPOnlyBasicProxyWithResolver(t *testing.T) { v := defaultVars() - v["NGINX_DNS_RESOLVER"] = "off" + v["NGINX_DNS_RESOLVER"] = "127.0.0.1:1053" out := renderTemplate(t, v) mustContain(t, out, "listen [::]:80;") - mustContain(t, out, "proxy_pass http://app-5000;") - mustContain(t, out, "upstream app-5000 {") - mustNotContain(t, out, "resolver 127.0.0.1:1053") - mustNotContain(t, out, "$dokku_upstream") + mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;") + mustContain(t, out, "resolver_timeout 5s;") + mustContain(t, out, `set $dokku_upstream "app.web.docker:5000";`) + mustContain(t, out, "proxy_pass http://$dokku_upstream;") + mustNotContain(t, out, "upstream app-5000 {") } func TestTemplate_HTTPSEmitsPushPreloadWhenSupported(t *testing.T) { @@ -155,7 +156,6 @@ func TestTemplate_HTTPRedirectsToHTTPSWhenSSLInUse(t *testing.T) { v["PROXY_PORT_MAP"] = "http:80:5000 https:443:5000" v["SSL_INUSE"] = "true" v["SSL_SERVER_NAME"] = "app.example.com" - v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) port80, _, ok := strings.Cut(out, "listen [::]:443") @@ -206,7 +206,6 @@ func TestTemplate_GRPCNoSSL(t *testing.T) { v := defaultVars() v["PROXY_PORT_MAP"] = "grpc:50051:50051" v["PROXY_UPSTREAM_PORTS"] = "50051" - v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "grpc_pass grpc://app-50051;") mustContain(t, out, "http2") @@ -215,6 +214,7 @@ func TestTemplate_GRPCNoSSL(t *testing.T) { func TestTemplate_GRPCNoSSLWithResolver(t *testing.T) { v := defaultVars() + v["NGINX_DNS_RESOLVER"] = "127.0.0.1:1053" v["PROXY_PORT_MAP"] = "grpc:50051:50051" v["PROXY_UPSTREAM_PORTS"] = "50051" out := renderTemplate(t, v) @@ -229,7 +229,6 @@ func TestTemplate_GRPCS(t *testing.T) { v["PROXY_UPSTREAM_PORTS"] = "50051" v["SSL_INUSE"] = "true" v["SSL_SERVER_NAME"] = "app.example.com" - v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "ssl_certificate /home/dokku/app/tls/server.crt;") mustContain(t, out, "grpc_pass grpc://app-50051;") @@ -262,7 +261,6 @@ func TestTemplate_UpstreamBlock(t *testing.T) { v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001" v["PROXY_UPSTREAM_PORTS"] = "5000 5001" v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000" - v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "upstream app-5000 {") mustContain(t, out, "upstream app-5001 {") @@ -274,6 +272,7 @@ func TestTemplate_UpstreamBlock(t *testing.T) { func TestTemplate_UpstreamBlockOmittedInResolverMode(t *testing.T) { v := defaultVars() + v["NGINX_DNS_RESOLVER"] = "127.0.0.1:1053" v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001" v["PROXY_UPSTREAM_PORTS"] = "5000 5001" v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000" @@ -286,12 +285,10 @@ func TestTemplate_UpstreamBlockOmittedInResolverMode(t *testing.T) { func TestTemplate_UpstreamWithKeepalive(t *testing.T) { v := defaultVars() v["PROXY_KEEPALIVE"] = "16" - v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "keepalive 16;") v2 := defaultVars() - v2["NGINX_DNS_RESOLVER"] = "off" out2 := renderTemplate(t, v2) mustNotContain(t, out2, "keepalive 16;") } @@ -335,6 +332,18 @@ func TestTemplate_RouteLocationAndUpstream(t *testing.T) { mustContain(t, out, "server 10.0.0.4:5001;") } +func TestTemplate_RouteUpstreamRenderedInResolverMode(t *testing.T) { + v := defaultVars() + v["NGINX_DNS_RESOLVER"] = "127.0.0.1:1053" + v["PROXY_ROUTES"] = "api|5001|/api/v0|0" + v["PROXY_ROUTE_LISTENERS"] = "api|5001|10.0.0.3:5001" + out := renderTemplate(t, v) + mustContain(t, out, "proxy_pass http://$dokku_upstream;") + mustContain(t, out, "proxy_pass http://app-api-5001;") + mustContain(t, out, "upstream app-api-5001 {") + mustNotContain(t, out, "upstream app-5000 {") +} + func TestTemplate_RouteStripPrefix(t *testing.T) { v := defaultVars() v["PROXY_ROUTES"] = "api|5001|/api/v0|1" diff --git a/plugins/nginx-vhosts/templates/nginx.conf.sigil b/plugins/nginx-vhosts/templates/nginx.conf.sigil index 5bb22703b..ad0bb149c 100644 --- a/plugins/nginx-vhosts/templates/nginx.conf.sigil +++ b/plugins/nginx-vhosts/templates/nginx.conf.sigil @@ -84,7 +84,7 @@ server { gzip_vary on; gzip_comp_level 6; -{{ if ne $.NGINX_DNS_RESOLVER "off" }} +{{ if $.NGINX_DNS_RESOLVER }} resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; resolver_timeout {{ $.NGINX_DNS_RESOLVER_TIMEOUT }}; set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; @@ -173,7 +173,7 @@ server { {{ end }} location / { -{{ if ne $.NGINX_DNS_RESOLVER "off" }} +{{ if $.NGINX_DNS_RESOLVER }} resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; resolver_timeout {{ $.NGINX_DNS_RESOLVER_TIMEOUT }}; set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; @@ -190,7 +190,7 @@ server { {{ end }} {{ end }} -{{ if eq $.NGINX_DNS_RESOLVER "off" }} +{{ if not $.NGINX_DNS_RESOLVER }} {{ if $.DOKKU_APP_WEB_LISTENERS }} {{ range $upstream_port := $.PROXY_UPSTREAM_PORTS | split " " }} upstream {{ $.APP }}-{{ $upstream_port }} { diff --git a/tests/test_coredns_outage b/tests/test_coredns_outage index 4b4ebc0f7..fe9b4b125 100755 --- a/tests/test_coredns_outage +++ b/tests/test_coredns_outage @@ -12,6 +12,7 @@ cleanup() { set +e start_coredns_docker ssh "$REMOTE" apps:destroy "$APP" --force >/dev/null 2>&1 + ssh "$REMOTE" nginx:set --global dns-resolver >/dev/null 2>&1 ssh "$REMOTE" events:off >/dev/null 2>&1 exit $exit_code } @@ -65,6 +66,7 @@ wait_for_status() { trap cleanup INT TERM EXIT ssh "$REMOTE" events:on +ssh "$REMOTE" nginx:set --global dns-resolver 127.0.0.1:1053 TMP=$(mktemp -d "/tmp/$TARGET.XXXXX") rmdir "$TMP" && cp -r "$(dirname "$SELF")/apps/python" "$TMP" diff --git a/tests/unit/nginx-vhosts_17.bats b/tests/unit/nginx-vhosts_17.bats index 2e9fe0865..d34145d06 100644 --- a/tests/unit/nginx-vhosts_17.bats +++ b/tests/unit/nginx-vhosts_17.bats @@ -17,12 +17,12 @@ teardown() { global_teardown } -@test "(nginx-vhosts:set) dns-resolver and dns-zone defaults report 127.0.0.1:1053 and docker" { +@test "(nginx-vhosts:set) dns-resolver defaults to empty and dns-zone defaults to docker" { run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-resolver" echo "output: $output" echo "status: $status" assert_success - assert_output "127.0.0.1:1053" + assert_output "" run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-zone" echo "output: $output" @@ -55,7 +55,27 @@ teardown() { assert_output "test" } -@test "(nginx-vhosts) generated nginx.conf uses resolver and variable-based proxy_pass by default" { +@test "(nginx-vhosts) generated nginx.conf uses IP-based upstream by default" { + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_contains "resolver " + assert_output_not_contains "\$dokku_upstream" + assert_output_contains "upstream ${TEST_APP}-" +} + +@test "(nginx-vhosts) generated nginx.conf uses resolver and variable-based proxy_pass when dns-resolver is set" { + run /bin/bash -c "dokku nginx:set --global dns-resolver 127.0.0.1:1053" + echo "output: $output" + echo "status: $status" + assert_success + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP echo "output: $output" echo "status: $status" @@ -72,8 +92,8 @@ teardown() { assert_output_not_contains "upstream ${TEST_APP}-" } -@test "(nginx-vhosts) generated nginx.conf falls back to IP-based upstream when dns-resolver=off" { - run /bin/bash -c "dokku nginx:set --global dns-resolver off" +@test "(nginx-vhosts) unsetting dns-resolver reverts to IP-based upstream" { + run /bin/bash -c "dokku nginx:set --global dns-resolver 127.0.0.1:1053" echo "output: $output" echo "status: $status" assert_success @@ -87,24 +107,7 @@ teardown() { echo "output: $output" echo "status: $status" assert_success - assert_output_not_contains "resolver 127.0.0.1:1053" - assert_output_not_contains "\$dokku_upstream" - assert_output_contains "upstream ${TEST_APP}-" -} - -@test "(nginx-vhosts) setting dns-resolver to empty re-engages the default resolver path" { - run /bin/bash -c "dokku nginx:set --global dns-resolver off" - echo "output: $output" - echo "status: $status" - assert_success - - run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP - echo "output: $output" - echo "status: $status" - assert_success - - run /bin/bash -c "dokku nginx:show-config $TEST_APP" - assert_output_contains "upstream ${TEST_APP}-" + assert_output_contains "resolver 127.0.0.1:1053 valid=10s ipv6=off" run /bin/bash -c "dokku nginx:set --global dns-resolver" echo "output: $output" @@ -120,11 +123,16 @@ teardown() { echo "output: $output" echo "status: $status" assert_success - assert_output_contains "resolver 127.0.0.1:1053 valid=10s ipv6=off" - assert_output_not_contains "upstream ${TEST_APP}-" + assert_output_not_contains "resolver 127.0.0.1:1053" + assert_output_contains "upstream ${TEST_APP}-" } @test "(nginx-vhosts) vendored nginx.conf.sigil renders IP-based upstream regardless of dns-resolver" { + run /bin/bash -c "dokku nginx:set --global dns-resolver 127.0.0.1:1053" + echo "output: $output" + echo "status: $status" + assert_success + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP custom_nginx_template echo "output: $output" echo "status: $status" @@ -140,6 +148,11 @@ teardown() { } @test "(nginx-vhosts) nginx returns 502 when coredns-docker is stopped" { + run /bin/bash -c "dokku nginx:set --global dns-resolver 127.0.0.1:1053" + echo "output: $output" + echo "status: $status" + assert_success + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP echo "output: $output" echo "status: $status" From 86175dd55c798ff4edde0fffd7a05cc7a69fee48 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 6 Nov 2026 22:06:27 -0500 Subject: [PATCH 07/10] fix: let the build set the coredns-docker version The minimum version is injected from `contrib/dependencies.json` at package build time, so hardcoding it in `debian/control` produced a duplicate constraint that `dpkg-deb` rejected. --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 7060a84c1..4d5e30a93 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.38.34 Section: web Priority: optional Architecture: amd64 -Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, coredns-docker (>= 0.6.0), docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux +Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, coredns-docker, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux Recommends: herokuish, bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, jq, nginx (>= 1.8.0) | openresty, bind9-dnsutils, debconf, plugn, sudo, python3, rsyslog Maintainer: Jose Diaz-Gonzalez From a46139eb315057fb96dd43d26dfc060240a6d69a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 6 Nov 2026 22:10:49 -0500 Subject: [PATCH 08/10] test: verify nginx returns 502 for a stopped app container --- tests/unit/nginx-vhosts_17.bats | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/unit/nginx-vhosts_17.bats b/tests/unit/nginx-vhosts_17.bats index d34145d06..0cf044171 100644 --- a/tests/unit/nginx-vhosts_17.bats +++ b/tests/unit/nginx-vhosts_17.bats @@ -175,3 +175,37 @@ teardown() { assert_http_localhost_response http "$TEST_APP.$DOKKU_DOMAIN" 80 / "" 200 } + +@test "(nginx-vhosts) nginx returns 502 when the app container is not running" { + run /bin/bash -c "dokku nginx:set --global dns-resolver 127.0.0.1:1053" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response http "$TEST_APP.$DOKKU_DOMAIN" 80 / "" 200 + + run /bin/bash -c "docker container stop $TEST_APP.web.1" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "proxy_pass http://\$dokku_upstream" + + assert_http_localhost_response http "$TEST_APP.$DOKKU_DOMAIN" 80 / "" 502 + + run /bin/bash -c "docker container start $TEST_APP.web.1" + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response http "$TEST_APP.$DOKKU_DOMAIN" 80 / "" 200 +} From ae8cbdcc6665f0e1291be2c46eb0483eebfbaee9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 6 Nov 2026 22:10:58 -0500 Subject: [PATCH 09/10] docs: require coredns-docker 0.6.1 --- docs/appendices/0.39.0-migration-guide.md | 2 +- docs/appendices/file-formats/nginx-conf-sigil.md | 2 +- docs/networking/proxies/nginx.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/appendices/0.39.0-migration-guide.md b/docs/appendices/0.39.0-migration-guide.md index c685b6ac0..43a710772 100644 --- a/docs/appendices/0.39.0-migration-guide.md +++ b/docs/appendices/0.39.0-migration-guide.md @@ -7,7 +7,7 @@ - The `--buildpacks-stack`, `--buildpacks-global-stack`, and `--buildpacks-computed-stack` report flags (and the corresponding `stack`, `global-stack`, `computed-stack` JSON keys) have been removed from `buildpacks:report`. Use `builder-herokuish:report` and `builder-pack:report` instead. The `buildpacks:report` command continues to report the buildpacks `list`. - The internal `buildpack-stack-name` plugin trigger has been removed. Each builder now reads its own `stack` property directly during the build. - Changing a builder's `stack` now clears that builder's build cache so the next deploy starts fresh. For herokuish this removes the `cache-` Docker volume; for pack the next `pack build` runs with `--clear-cache`. -- The `dokku` Debian package now declares `coredns-docker (>= 0.6.0)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. +- The `dokku` Debian package now declares `coredns-docker (>= 0.6.1)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. - The bundled `nginx.conf.sigil` can now use an nginx `resolver` directive and a variable-based `proxy_pass` against `..docker` instead of pinning the upstream to a literal IP. This addresses the long-standing post-reboot and post-`apt-upgrade-docker` stale-vhost bug ([#8313](https://github.com/dokku/dokku/issues/8313)) - the rendered upstream re-resolves at request time via coredns-docker, so a host reboot or single-container restart no longer leaves nginx routing to a stale IP. This behavior is opt-in, and upstreams remain IP-based unless `dns-resolver` is set: ```shell diff --git a/docs/appendices/file-formats/nginx-conf-sigil.md b/docs/appendices/file-formats/nginx-conf-sigil.md index c0324cacf..2403fd019 100644 --- a/docs/appendices/file-formats/nginx-conf-sigil.md +++ b/docs/appendices/file-formats/nginx-conf-sigil.md @@ -96,7 +96,7 @@ This path renders the IP that Dokku saw when the config was last built. After a ### DNS-based upstream (resolver path) -The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_RESOLVER_TIMEOUT` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.6.0`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. +The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_RESOLVER_TIMEOUT` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.6.1`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. The bundled template emits this snippet: diff --git a/docs/networking/proxies/nginx.md b/docs/networking/proxies/nginx.md index 1e07b6758..fcef00e4b 100644 --- a/docs/networking/proxies/nginx.md +++ b/docs/networking/proxies/nginx.md @@ -35,7 +35,7 @@ Nginx will proxy the requests in a [round-robin balancing fashion](http://nginx. Dokku can resolve nginx upstreams through [coredns-docker](https://github.com/dokku/coredns-docker), a CoreDNS plugin that watches the Docker daemon and serves fresh `A` records keyed off container labels. When enabled, the nginx `resolver` directive points at coredns-docker's local socket (`127.0.0.1:1053`) and the rendered upstream is a variable-based `proxy_pass`, which forces nginx to re-resolve at request time honoring DNS TTL. This is how nginx avoids serving the wrong vhost after a host reboot, after `apt upgrade docker`, or after a single container restarts mid-runtime: the on-disk nginx config no longer pins a stale IP. This behavior is opt-in, and nginx uses IP-based `upstream` blocks by default. -`coredns-docker` (>= `0.6.0`) is installed automatically as a transitive `Depends:` of the `dokku` Debian package, so `apt install dokku` and `apt upgrade dokku` pull it in without operator action. The shipped `/etc/coredns/Corefile` already binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from Dokku's existing `com.dokku.app-name` and `com.dokku.process-type` container labels. Multiple containers for the same process type collapse onto one multi-A record set, so scaled apps load-balance across instances via DNS without any per-container configuration. The bundled Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. +`coredns-docker` (>= `0.6.1`) is installed automatically as a transitive `Depends:` of the `dokku` Debian package, so `apt install dokku` and `apt upgrade dokku` pull it in without operator action. The shipped `/etc/coredns/Corefile` already binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from Dokku's existing `com.dokku.app-name` and `com.dokku.process-type` container labels. Multiple containers for the same process type collapse onto one multi-A record set, so scaled apps load-balance across instances via DNS without any per-container configuration. The bundled Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. No `systemd-resolved` or `/etc/resolv.conf` change is required - nginx talks to `coredns-docker` directly via the `resolver` directive in each app's `nginx.conf`. From a0c790c353b5d5491d54661c94916d2fb4209ca8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 7 Nov 2026 03:18:52 -0500 Subject: [PATCH 10/10] fix: require coredns-docker 0.8.0 The previous release's default Corefile held host port 8080 on every interface, which broke apps and proxies mapping that port. 0.8.0 binds all of its listeners to `127.0.0.1` and moves the health and ready endpoints off common ports. --- contrib/dependencies.json | 6 +++--- docs/appendices/0.39.0-migration-guide.md | 2 +- docs/appendices/file-formats/nginx-conf-sigil.md | 2 +- docs/networking/proxies/nginx.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/dependencies.json b/contrib/dependencies.json index b4561830f..f68379c30 100644 --- a/contrib/dependencies.json +++ b/contrib/dependencies.json @@ -2,10 +2,10 @@ "dependencies": [ { "name": "coredns-docker", - "version": "0.6.1", + "version": "0.8.0", "urls": { - "amd64": "https://github.com/dokku/coredns-docker/releases/download/v0.6.1/coredns-docker-linux-amd64", - "arm64": "https://github.com/dokku/coredns-docker/releases/download/v0.6.1/coredns-docker-linux-arm64" + "amd64": "https://github.com/dokku/coredns-docker/releases/download/0.8.0/coredns-docker-linux-amd64", + "arm64": "https://github.com/dokku/coredns-docker/releases/download/0.8.0/coredns-docker-linux-arm64" } }, { diff --git a/docs/appendices/0.39.0-migration-guide.md b/docs/appendices/0.39.0-migration-guide.md index 43a710772..5d0165bce 100644 --- a/docs/appendices/0.39.0-migration-guide.md +++ b/docs/appendices/0.39.0-migration-guide.md @@ -7,7 +7,7 @@ - The `--buildpacks-stack`, `--buildpacks-global-stack`, and `--buildpacks-computed-stack` report flags (and the corresponding `stack`, `global-stack`, `computed-stack` JSON keys) have been removed from `buildpacks:report`. Use `builder-herokuish:report` and `builder-pack:report` instead. The `buildpacks:report` command continues to report the buildpacks `list`. - The internal `buildpack-stack-name` plugin trigger has been removed. Each builder now reads its own `stack` property directly during the build. - Changing a builder's `stack` now clears that builder's build cache so the next deploy starts fresh. For herokuish this removes the `cache-` Docker volume; for pack the next `pack build` runs with `--clear-cache`. -- The `dokku` Debian package now declares `coredns-docker (>= 0.6.1)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. +- The `dokku` Debian package now declares `coredns-docker (>= 0.8.0)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. - The bundled `nginx.conf.sigil` can now use an nginx `resolver` directive and a variable-based `proxy_pass` against `..docker` instead of pinning the upstream to a literal IP. This addresses the long-standing post-reboot and post-`apt-upgrade-docker` stale-vhost bug ([#8313](https://github.com/dokku/dokku/issues/8313)) - the rendered upstream re-resolves at request time via coredns-docker, so a host reboot or single-container restart no longer leaves nginx routing to a stale IP. This behavior is opt-in, and upstreams remain IP-based unless `dns-resolver` is set: ```shell diff --git a/docs/appendices/file-formats/nginx-conf-sigil.md b/docs/appendices/file-formats/nginx-conf-sigil.md index 2403fd019..f8aca1267 100644 --- a/docs/appendices/file-formats/nginx-conf-sigil.md +++ b/docs/appendices/file-formats/nginx-conf-sigil.md @@ -96,7 +96,7 @@ This path renders the IP that Dokku saw when the config was last built. After a ### DNS-based upstream (resolver path) -The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_RESOLVER_TIMEOUT` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.6.1`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. +The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_RESOLVER_TIMEOUT` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.8.0`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. The bundled template emits this snippet: diff --git a/docs/networking/proxies/nginx.md b/docs/networking/proxies/nginx.md index fcef00e4b..c4530c31e 100644 --- a/docs/networking/proxies/nginx.md +++ b/docs/networking/proxies/nginx.md @@ -35,7 +35,7 @@ Nginx will proxy the requests in a [round-robin balancing fashion](http://nginx. Dokku can resolve nginx upstreams through [coredns-docker](https://github.com/dokku/coredns-docker), a CoreDNS plugin that watches the Docker daemon and serves fresh `A` records keyed off container labels. When enabled, the nginx `resolver` directive points at coredns-docker's local socket (`127.0.0.1:1053`) and the rendered upstream is a variable-based `proxy_pass`, which forces nginx to re-resolve at request time honoring DNS TTL. This is how nginx avoids serving the wrong vhost after a host reboot, after `apt upgrade docker`, or after a single container restarts mid-runtime: the on-disk nginx config no longer pins a stale IP. This behavior is opt-in, and nginx uses IP-based `upstream` blocks by default. -`coredns-docker` (>= `0.6.1`) is installed automatically as a transitive `Depends:` of the `dokku` Debian package, so `apt install dokku` and `apt upgrade dokku` pull it in without operator action. The shipped `/etc/coredns/Corefile` already binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from Dokku's existing `com.dokku.app-name` and `com.dokku.process-type` container labels. Multiple containers for the same process type collapse onto one multi-A record set, so scaled apps load-balance across instances via DNS without any per-container configuration. The bundled Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. +`coredns-docker` (>= `0.8.0`) is installed automatically as a transitive `Depends:` of the `dokku` Debian package, so `apt install dokku` and `apt upgrade dokku` pull it in without operator action. The shipped `/etc/coredns/Corefile` already binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from Dokku's existing `com.dokku.app-name` and `com.dokku.process-type` container labels. Multiple containers for the same process type collapse onto one multi-A record set, so scaled apps load-balance across instances via DNS without any per-container configuration. The bundled Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. No `systemd-resolved` or `/etc/resolv.conf` change is required - nginx talks to `coredns-docker` directly via the `resolver` directive in each app's `nginx.conf`.