From 5241fd4be2efd8e420fe5210feb03ff7645d6bee Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 6 Nov 2026 20:43:16 -0500 Subject: [PATCH] feat: resolve nginx upstreams via coredns-docker Stops nginx from routing to stale container IPs after host reboots, Docker upgrades without live-restore, or runtime container restarts. The bundled `nginx.conf.sigil` can use an nginx `resolver` directive against the local `coredns-docker` socket and a variable-based `proxy_pass` keyed on `..docker`, so the upstream is resolved at request time honoring DNS TTL. `coredns-docker` becomes a hard `Depends:` of the dokku deb and is bundled in the Docker image under runit. Closes #8313. --- contrib/dependencies.json | 8 + debian/control | 2 +- .../runit/runsvdir/default/coredns-docker/run | 7 + docs/appendices/0.39.0-migration-guide.md | 16 ++ .../file-formats/nginx-conf-sigil.md | 51 ++++++- docs/getting-started/install/docker.md | 1 + docs/networking/proxies/nginx.md | 28 ++++ plugins/nginx-vhosts/functions | 20 ++- plugins/nginx-vhosts/internal-functions | 42 ++++++ plugins/nginx-vhosts/nginx_vhosts.go | 34 +++++ plugins/nginx-vhosts/report.go | 2 + .../src/nginx-property/nginx-property.go | 12 ++ plugins/nginx-vhosts/subcommands/set | 6 +- plugins/nginx-vhosts/template_test.go | 47 +++++- .../nginx-vhosts/templates/nginx.conf.sigil | 15 +- tests/unit/nginx-vhosts_17.bats | 139 ++++++++++++++++++ 16 files changed, 419 insertions(+), 11 deletions(-) create mode 100755 docker/etc/runit/runsvdir/default/coredns-docker/run create mode 100644 tests/unit/nginx-vhosts_17.bats diff --git a/contrib/dependencies.json b/contrib/dependencies.json index a2c7a3931..b4561830f 100644 --- a/contrib/dependencies.json +++ b/contrib/dependencies.json @@ -1,5 +1,13 @@ { "dependencies": [ + { + "name": "coredns-docker", + "version": "0.6.1", + "urls": { + "amd64": "https://github.com/dokku/coredns-docker/releases/download/v0.6.1/coredns-docker-linux-amd64", + "arm64": "https://github.com/dokku/coredns-docker/releases/download/v0.6.1/coredns-docker-linux-arm64" + } + }, { "name": "docker-container-healthchecker", "version": "0.16.1", diff --git a/debian/control b/debian/control index 703edb864..4d5e30a93 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.38.34 Section: web Priority: optional Architecture: amd64 -Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux +Depends: apache2-utils, locales, git, cpio, cron | cron-daemon, curl, logrotate, man-db, netcat, sshcommand, docker-engine-cs (>= 19.03.0) | docker-engine (>= 19.03.0) | docker-io (>= 19.03.0) | docker.io (>= 19.03.0) | docker-ce (>= 19.03.0) | docker-ee (>= 19.03.0) | moby-engine, docker-compose-plugin | moby-compose | docker-compose, docker-buildx-plugin | moby-buildx | docker-buildx, coredns-docker, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, parallel, procfile-util, rsync, dos2unix, unzip, util-linux Recommends: herokuish, bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, jq, nginx (>= 1.8.0) | openresty, bind9-dnsutils, debconf, plugn, sudo, python3, rsyslog Maintainer: Jose Diaz-Gonzalez diff --git a/docker/etc/runit/runsvdir/default/coredns-docker/run b/docker/etc/runit/runsvdir/default/coredns-docker/run new file mode 100755 index 000000000..312b939d2 --- /dev/null +++ b/docker/etc/runit/runsvdir/default/coredns-docker/run @@ -0,0 +1,7 @@ +#!/bin/bash + +exec >>/var/log/services/coredns-docker +exec 2>&1 + +echo "Running coredns-docker" +exec /usr/bin/coredns-docker -conf /etc/coredns/Corefile diff --git a/docs/appendices/0.39.0-migration-guide.md b/docs/appendices/0.39.0-migration-guide.md index 420e5f7b4..4b40ebe05 100644 --- a/docs/appendices/0.39.0-migration-guide.md +++ b/docs/appendices/0.39.0-migration-guide.md @@ -7,6 +7,22 @@ - The `--buildpacks-stack`, `--buildpacks-global-stack`, and `--buildpacks-computed-stack` report flags (and the corresponding `stack`, `global-stack`, `computed-stack` JSON keys) have been removed from `buildpacks:report`. Use `builder-herokuish:report` and `builder-pack:report` instead. The `buildpacks:report` command continues to report the buildpacks `list`. - The internal `buildpack-stack-name` plugin trigger has been removed. Each builder now reads its own `stack` property directly during the build. - Changing a builder's `stack` now clears that builder's build cache so the next deploy starts fresh. For herokuish this removes the `cache-` Docker volume; for pack the next `pack build` runs with `--clear-cache`. +- The `dokku` Debian package now declares `coredns-docker (>= 0.6.0)` as a hard `Depends:`. `apt install dokku` and `apt upgrade dokku` pull it in automatically. The deb ships a default `/etc/coredns/Corefile` that binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from the `com.dokku.app-name` and `com.dokku.process-type` labels Dokku already stamps on every deployed container. The Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. +- The bundled `nginx.conf.sigil` now uses an nginx `resolver` directive and a variable-based `proxy_pass` against `..docker` instead of pinning the upstream to a literal IP. This is what addresses the long-standing post-reboot and post-`apt-upgrade-docker` stale-vhost bug ([#8313](https://github.com/dokku/dokku/issues/8313)) - the rendered upstream re-resolves at request time via coredns-docker, so a host reboot or single-container restart no longer leaves nginx routing to a stale IP. See the [nginx documentation](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for details. +- Two new global nginx properties control the resolver path: + + - `dns-resolver` (default: `127.0.0.1:1053`) - the address nginx talks to. + - `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. + + Both can be set via `dokku nginx:set --global `. +- The literal string `off` is the sentinel for disabling the resolver path. **Setting `dns-resolver` to the empty string does not disable it** - that deletes the property file and the new default (`127.0.0.1:1053`) re-engages. To revert to the legacy IP-based upstream block on hosts where coredns-docker cannot run (source/git installs, locally held coredns-docker package, postinst failures): + + ```shell + dokku nginx:set --global dns-resolver off + dokku proxy:build-config --all + ``` + +- Vendored `nginx.conf.sigil` templates are unaffected by the change. The existing `DOKKU_APP__LISTENERS` variable continues to be populated with the IP-based listener list exactly as before, so templates that only reference it continue to render IP-based upstreams regardless of the new property defaults. To opt a vendored template into the resolver path, switch its `proxy_pass` line to use the new `DOKKU_APP__LISTENER_HOST`, `NGINX_DNS_RESOLVER`, and `NGINX_DNS_ZONE` sigil variables - see the [nginx.conf.sigil reference](/docs/appendices/file-formats/nginx-conf-sigil.md#upstream-rendering-ip-based-vs-dns-based) for the exact snippet. ### Stack property migrated to builder plugins diff --git a/docs/appendices/file-formats/nginx-conf-sigil.md b/docs/appendices/file-formats/nginx-conf-sigil.md index 78fba3f58..bb9a063f6 100644 --- a/docs/appendices/file-formats/nginx-conf-sigil.md +++ b/docs/appendices/file-formats/nginx-conf-sigil.md @@ -4,7 +4,10 @@ The `nginx.conf.sigil` file is used to configure the nginx server for an applica ## Validation -A custom `nginx.conf.sigil` is pre-validated at the start of every deploy, immediately after it is extracted from the source tree and before the build phase runs. Pre-validation renders the template via sigil with the same parameters used at deploy time, wraps the rendered config in a minimal `events`/`http` scaffold, and runs `nginx -t` against the result. The deploy is aborted if either the sigil render or the `nginx -t` check fails, so build work is not wasted on a syntactically invalid template. When no app listeners exist yet (typical for first deploys), pre-validation injects a placeholder `127.0.0.1:5000` listener for `DOKKU_APP_WEB_LISTENERS` so that the rendered upstream block has a static server entry and `nginx -t` does not bail out on "host not found in upstream". +A custom `nginx.conf.sigil` is pre-validated at the start of every deploy, immediately after it is extracted from the source tree and before the build phase runs. Pre-validation renders the template via sigil with the same parameters used at deploy time, wraps the rendered config in a minimal `events`/`http` scaffold, and runs `nginx -t` against the result. The deploy is aborted if either the sigil render or the `nginx -t` check fails, so build work is not wasted on a syntactically invalid template. When no app listeners exist yet (typical for first deploys), pre-validation injects placeholders so the rendered config parses cleanly under `nginx -t`: + +- `DOKKU_APP_WEB_LISTENERS` is set to `127.0.0.1:5000` so the legacy `upstream` block has a static server entry. +- `DOKKU_APP_WEB_LISTENER_HOST` is set to `127.0.0.1` so the resolver-based `set $dokku_upstream "...";` line renders to a syntactically valid value. Pre-validation is skipped when the proxy type is not `nginx` or when `disable-custom-config` is set to `true` for the app. @@ -68,3 +71,49 @@ listen {{ if $.NGINX_BIND_ADDRESS_IP4 }}{{ $.NGINX_BIND_ADDRESS_IP4 }}:{{en ``` Dokku logs a deprecation warning during deploys when a custom template still uses the `listen ... http2` form, so the offending template can be located via the deploy output. + +## Upstream Rendering: IP-based vs DNS-based + +> [!IMPORTANT] +> New as of 0.39.0 + +The bundled `nginx.conf.sigil` supports two rendering paths for the upstream, and both sets of sigil variables are populated on every render. Vendored templates can use either path independently of the global `dns-resolver` setting. + +### IP-based upstream (legacy path) + +The classic form uses `DOKKU_APP__LISTENERS`: a space-separated `:` list, one entry per scaled instance of the process type. The IPs come from `/home/dokku//IP..` and the ports from `/home/dokku//PORT..`. A typical use is an `upstream` block with one `server :;` per instance: + +```nginx +upstream {{ $.APP }}-{{ $upstream_port }} { +{{ range $listeners := $.DOKKU_APP_WEB_LISTENERS | split " " }} +{{ $listener_list := $listeners | split ":" }} +{{ $listener_ip := index $listener_list 0 }} + server {{ $listener_ip }}:{{ $upstream_port }};{{ end }} +} +``` + +This path renders the IP that Dokku saw when the config was last built. After a host reboot, an `apt upgrade docker` without live-restore, or a single container restart, those IPs can become stale until `dokku proxy:build-config` runs again. The DNS-based path below addresses that staleness. + +### DNS-based upstream (resolver path) + +The new variables (`DOKKU_APP__LISTENER_HOST` + `NGINX_DNS_RESOLVER` + `NGINX_DNS_ZONE`) describe a single hostname per process type of the form `..` (for example, `myapp.web.docker`). The hostname is served by [coredns-docker](https://github.com/dokku/coredns-docker) (>= `0.6.0`), which watches Docker events and collapses every running instance of the process type onto one multi-A record set. Combined with an nginx `resolver` directive and a variable-based `proxy_pass`, the upstream re-resolves at request time honoring DNS TTL - which gives runtime IP refresh on container restarts and DNS round-robin across scaled instances. + +The bundled template emits this snippet: + +```nginx +{{ if ne $.NGINX_DNS_RESOLVER "off" }} +resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; +set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; +proxy_pass http://$dokku_upstream; +{{ else }} +proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; +{{ end }} +``` + +The `$dokku_upstream` variable name is intentionally namespaced (rather than `$upstream`) so vendored templates that already define `$upstream` will not collide if they adopt the resolver branch. + +`NGINX_DNS_RESOLVER` defaults to `127.0.0.1:1053` and `NGINX_DNS_ZONE` defaults to `docker`; both are configurable via `dokku nginx:set --global dns-resolver ...` and `dokku nginx:set --global dns-zone ...`. The literal string `off` is the sentinel for disabling the resolver branch in the bundled template - empty strings are treated as "use the default" because the property is stored on disk and an empty value deletes the property file. + +### Choosing between the two + +The DNS-based path is the default for the bundled template. Vendored templates that have not been updated continue to use `DOKKU_APP__LISTENERS` and are unaffected by the global default. The DNS-based variables are populated regardless of whether `dns-resolver` is `off` or not, so a vendored template can adopt the resolver path on its own schedule by switching its rendering snippet, even if other apps on the host are still using IP-based upstreams. diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md index b336b21de..15d7efbe7 100644 --- a/docs/getting-started/install/docker.md +++ b/docs/getting-started/install/docker.md @@ -88,6 +88,7 @@ Dokku is run in the following configuration: - Image build cache is set to the data dir + `/home/dokku`. - The docker socket is mounted into container. - `DOKKU_HOST_ROOT` and `DOKKU_LIB_HOST_ROOT` tell Dokku how the host Docker daemon sees those same trees when creating bind mounts (app repos and plugin data such as named storage). +- `coredns-docker` (a CoreDNS plugin that serves DNS records for sibling Docker containers) is installed transitively via the dokku deb's `Depends:`, runs under runit on `127.0.0.1:1053` inside the container, and is what the `nginx-vhosts` `dns-resolver` default points at. This is what makes the `..docker` upstream resolution work out of the box - see the [nginx proxy doc](/docs/networking/proxies/nginx.md#upstream-resolution-via-coredns-docker) for the operator-facing behavior. Application repositories, plugin config, as well as plugin data are persisted to disk within the specified host directory for `/var/lib/dokku`. `DOKKU_LIB_HOST_ROOT` must point at that host path's `var/lib/dokku` subtree so Docker `-v` mounts resolve under the data volume rather than under the host's own `/var/lib/dokku`. diff --git a/docs/networking/proxies/nginx.md b/docs/networking/proxies/nginx.md index 4ac343db6..8f5e0ab63 100644 --- a/docs/networking/proxies/nginx.md +++ b/docs/networking/proxies/nginx.md @@ -28,6 +28,34 @@ Nginx will proxy the requests in a [round-robin balancing fashion](http://nginx. > [!NOTE] > Due to how the plugin is implemented, if an app successfully starts up `web` containers but fails to deploy some other containers, nginx may eventually stop routing requests. Users should revert their code in these cases, or manually trigger `dokku proxy:build-config $APP` in order to ensure requests route to the new web containers. +### Upstream Resolution via coredns-docker + +> [!IMPORTANT] +> New as of 0.39.0 + +Dokku resolves nginx upstreams through [coredns-docker](https://github.com/dokku/coredns-docker), a CoreDNS plugin that watches the Docker daemon and serves fresh `A` records keyed off container labels. The nginx `resolver` directive points at coredns-docker's local socket (`127.0.0.1:1053`) and the rendered upstream is a variable-based `proxy_pass`, which forces nginx to re-resolve at request time honoring DNS TTL. This is how nginx avoids serving the wrong vhost after a host reboot, after `apt upgrade docker`, or after a single container restarts mid-runtime: the on-disk nginx config no longer pins a stale IP. + +`coredns-docker` (>= `0.6.0`) is installed automatically as a transitive `Depends:` of the `dokku` Debian package, so `apt install dokku` and `apt upgrade dokku` pull it in without operator action. The shipped `/etc/coredns/Corefile` already binds `127.0.0.1:1053`, serves the `docker.` zone, and synthesizes `..docker` and `.docker` `A` records from Dokku's existing `com.dokku.app-name` and `com.dokku.process-type` container labels. Multiple containers for the same process type collapse onto one multi-A record set, so scaled apps load-balance across instances via DNS without any per-container configuration. The bundled Docker image of Dokku ships coredns-docker preinstalled and runs it under runit on the same port. + +No `systemd-resolved` or `/etc/resolv.conf` change is required - nginx talks to `coredns-docker` directly via the `resolver` directive in each app's `nginx.conf`. + +The behavior is controlled by two global nginx properties: + +- `dns-resolver` (default: `127.0.0.1:1053`) - the resolver address nginx talks to. +- `dns-zone` (default: `docker`) - the suffix appended to `.` when constructing the upstream hostname. + +To revert to the legacy IP-based upstream block (for instance on a host where `coredns-docker` cannot be installed): + +```shell +dokku nginx:set --global dns-resolver off +dokku proxy:build-config --all +``` + +> [!NOTE] +> The sentinel for disabling the feature is the literal string `off`. Setting `dns-resolver` to the empty string (`""`) does **not** disable it - that deletes the property and the default (`127.0.0.1:1053`) re-engages. Use `off` to disable, the default to re-enable. + +See [the `nginx.conf.sigil` reference](/docs/appendices/file-formats/nginx-conf-sigil.md) for the sigil variables that drive the resolver-based render, and how vendored templates can adopt or skip it. + ### Nginx Configuration for Undeployed Apps > [!IMPORTANT] diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index 6426c4b72..051a8f525 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -324,6 +324,8 @@ fn-nginx-vhosts-render-template() { local NGINX_BIND_ADDRESS_IP4="$(fn-nginx-computed-bind-address-ipv4 "$APP")" local NGINX_BIND_ADDRESS_IP6="$(fn-nginx-computed-bind-address-ipv6 "$APP")" + local NGINX_DNS_RESOLVER="$(fn-nginx-computed-dns-resolver "$APP")" + local NGINX_DNS_ZONE="$(fn-nginx-computed-dns-zone "$APP")" local NGINX_UNDERSCORE_IN_HEADERS="$(fn-nginx-computed-underscore-in-headers "$APP")" local PROXY_X_FORWARDED_FOR="$(fn-nginx-computed-x-forwarded-for-value "$APP")" local PROXY_X_FORWARDED_PORT="$(fn-nginx-computed-x-forwarded-port-value "$APP")" @@ -354,6 +356,8 @@ fn-nginx-vhosts-render-template() { NGINX_ERROR_LOG_PATH="$NGINX_ERROR_LOG_PATH" NGINX_BIND_ADDRESS_IP4="$NGINX_BIND_ADDRESS_IP4" NGINX_BIND_ADDRESS_IP6="$NGINX_BIND_ADDRESS_IP6" + NGINX_DNS_RESOLVER="$NGINX_DNS_RESOLVER" + NGINX_DNS_ZONE="$NGINX_DNS_ZONE" HTTP2_DIRECTIVE_SUPPORTED="$HTTP2_DIRECTIVE_SUPPORTED" HTTP2_PUSH_SUPPORTED="$HTTP2_PUSH_SUPPORTED" GRPC_SUPPORTED="$GRPC_SUPPORTED" @@ -386,21 +390,29 @@ fn-nginx-vhosts-render-template() { PROXY_X_FORWARDED_SSL="$PROXY_X_FORWARDED_SSL") if [[ "$IS_DEPLOYED_WITH_LISTENERS" == "true" ]]; then - local line PROC_TYPE LISTENERS UPP_PROC_TYPE + local line PROC_TYPE LISTENERS UPP_PROC_TYPE LISTENER_HOST while read -r line || [[ -n "$line" ]]; do PROC_TYPE=${line%%=*} LISTENERS="$(plugn trigger network-get-listeners "$APP" "$PROC_TYPE" | xargs)" UPP_PROC_TYPE="${PROC_TYPE^^}" UPP_PROC_TYPE="${UPP_PROC_TYPE//-/_}" SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENERS=$LISTENERS") + LISTENER_HOST="" + if [[ -n "$NGINX_DNS_ZONE" ]]; then + LISTENER_HOST="${APP}.${PROC_TYPE}.${NGINX_DNS_ZONE}" + fi + SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENER_HOST=$LISTENER_HOST") done < <(plugn trigger ps-current-scale "$APP") elif [[ "$PRE_VALIDATE" == "true" ]]; then - # During pre-validation no listeners exist yet; supply a placeholder so - # any upstream block in the template renders with a static server entry - # and nginx -t does not bail out on "host not found in upstream". + # During pre-validation no listeners exist yet; supply placeholders so + # both the legacy IP-based upstream block and the DNS-based resolver + # branch render syntactically valid nginx config that nginx -t can + # parse (no "host not found in upstream" and no empty set $var). SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENERS="127.0.0.1:5000") + SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENER_HOST="127.0.0.1") else SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENERS="") + SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENER_HOST="") fi sigil "${SIGIL_PARAMS[@]}" | cat -s >"$OUTPUT_PATH" diff --git a/plugins/nginx-vhosts/internal-functions b/plugins/nginx-vhosts/internal-functions index a2616eb84..3b9e378eb 100755 --- a/plugins/nginx-vhosts/internal-functions +++ b/plugins/nginx-vhosts/internal-functions @@ -172,6 +172,48 @@ fn-nginx-global-disable-custom-config() { fn-get-property --app "$APP" --global "disable-custom-config" } +fn-nginx-dns-resolver() { + declare desc="get the configured dns-resolver value" + declare APP="$1" + + fn-get-property --app "$APP" "dns-resolver" +} + +fn-nginx-computed-dns-resolver() { + declare desc="get the computed dns-resolver value" + declare APP="$1" + + fn-get-property --app "$APP" --computed "dns-resolver" +} + +fn-nginx-global-dns-resolver() { + declare desc="get the global dns-resolver value" + declare APP="$1" + + fn-get-property --app "$APP" --global "dns-resolver" +} + +fn-nginx-dns-zone() { + declare desc="get the configured dns-zone value" + declare APP="$1" + + fn-get-property --app "$APP" "dns-zone" +} + +fn-nginx-computed-dns-zone() { + declare desc="get the computed dns-zone value" + declare APP="$1" + + fn-get-property --app "$APP" --computed "dns-zone" +} + +fn-nginx-global-dns-zone() { + declare desc="get the global dns-zone value" + declare APP="$1" + + fn-get-property --app "$APP" --global "dns-zone" +} + fn-nginx-error-log-path() { declare desc="get the configured error log path" declare APP="$1" diff --git a/plugins/nginx-vhosts/nginx_vhosts.go b/plugins/nginx-vhosts/nginx_vhosts.go index 7a51f6e10..bd6b14cae 100644 --- a/plugins/nginx-vhosts/nginx_vhosts.go +++ b/plugins/nginx-vhosts/nginx_vhosts.go @@ -150,6 +150,40 @@ func AppDisableCustomConfig(appName string) string { return common.PropertyGet("nginx", appName, "disable-custom-config") } +func AppDnsResolver(appName string) string { + return common.PropertyGet("nginx", appName, "dns-resolver") +} + +func ComputedDnsResolver(appName string) string { + appValue := AppDnsResolver(appName) + if appValue != "" { + return appValue + } + + return GlobalDnsResolver() +} + +func GlobalDnsResolver() string { + return common.PropertyGetDefault("nginx", "--global", "dns-resolver", "127.0.0.1:1053") +} + +func AppDnsZone(appName string) string { + return common.PropertyGet("nginx", appName, "dns-zone") +} + +func ComputedDnsZone(appName string) string { + appValue := AppDnsZone(appName) + if appValue != "" { + return appValue + } + + return GlobalDnsZone() +} + +func GlobalDnsZone() string { + return common.PropertyGetDefault("nginx", "--global", "dns-zone", "docker") +} + func ComputedDisableCustomConfig(appName string) string { appValue := AppDisableCustomConfig(appName) if appValue != "" { diff --git a/plugins/nginx-vhosts/report.go b/plugins/nginx-vhosts/report.go index 3cf18eb3c..f6d29bd68 100644 --- a/plugins/nginx-vhosts/report.go +++ b/plugins/nginx-vhosts/report.go @@ -29,6 +29,8 @@ func nginxReportProps() []nginxReportProp { {"client-header-timeout", AppClientHeaderTimeout, ComputedClientHeaderTimeout, wrapGlobal(GlobalClientHeaderTimeout)}, {"client-max-body-size", AppClientMaxBodySize, ComputedClientMaxBodySize, wrapGlobal(GlobalClientMaxBodySize)}, {"disable-custom-config", AppDisableCustomConfig, ComputedDisableCustomConfig, wrapGlobal(GlobalDisableCustomConfig)}, + {"dns-resolver", AppDnsResolver, ComputedDnsResolver, wrapGlobal(GlobalDnsResolver)}, + {"dns-zone", AppDnsZone, ComputedDnsZone, wrapGlobal(GlobalDnsZone)}, {"error-log-path", AppErrorLogPath, ComputedErrorLogPath, wrapGlobal(GlobalErrorLogPath)}, {"hsts-include-subdomains", AppHSTSIncludeSubdomains, ComputedHSTSIncludeSubdomains, wrapGlobal(GlobalHSTSIncludeSubdomains)}, {"hsts-max-age", AppHSTSMaxAge, ComputedHSTSMaxAge, wrapGlobal(GlobalHSTSMaxAge)}, diff --git a/plugins/nginx-vhosts/src/nginx-property/nginx-property.go b/plugins/nginx-vhosts/src/nginx-property/nginx-property.go index 98b719b4d..bd1e3c28b 100644 --- a/plugins/nginx-vhosts/src/nginx-property/nginx-property.go +++ b/plugins/nginx-vhosts/src/nginx-property/nginx-property.go @@ -51,6 +51,10 @@ func appValue(appName string, property string) string { value = nginx_vhosts.AppClientHeaderTimeout(appName) case "disable-custom-config": value = nginx_vhosts.AppDisableCustomConfig(appName) + case "dns-resolver": + value = nginx_vhosts.AppDnsResolver(appName) + case "dns-zone": + value = nginx_vhosts.AppDnsZone(appName) case "error-log-path": value = nginx_vhosts.AppErrorLogPath(appName) case "hsts-include-subdomains": @@ -123,6 +127,10 @@ func computedValue(appName string, property string) string { value = nginx_vhosts.ComputedClientMaxBodySize(appName) case "disable-custom-config": value = nginx_vhosts.ComputedDisableCustomConfig(appName) + case "dns-resolver": + value = nginx_vhosts.ComputedDnsResolver(appName) + case "dns-zone": + value = nginx_vhosts.ComputedDnsZone(appName) case "error-log-path": value = nginx_vhosts.ComputedErrorLogPath(appName) case "hsts-include-subdomains": @@ -195,6 +203,10 @@ func globalValue(appName string, property string) string { value = nginx_vhosts.GlobalClientMaxBodySize() case "disable-custom-config": value = nginx_vhosts.GlobalDisableCustomConfig() + case "dns-resolver": + value = nginx_vhosts.GlobalDnsResolver() + case "dns-zone": + value = nginx_vhosts.GlobalDnsZone() case "error-log-path": value = nginx_vhosts.GlobalErrorLogPath() case "hsts-include-subdomains": diff --git a/plugins/nginx-vhosts/subcommands/set b/plugins/nginx-vhosts/subcommands/set index ee8ad3530..439d421d4 100755 --- a/plugins/nginx-vhosts/subcommands/set +++ b/plugins/nginx-vhosts/subcommands/set @@ -9,13 +9,13 @@ cmd-nginx-set() { declare cmd="nginx:set" [[ "$1" == "$cmd" ]] && shift 1 declare APP="$1" KEY="$2" VALUE="$3" - local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") - local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") + local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") + local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl") [[ -z "$KEY" ]] && dokku_log_fail "No key specified" if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then - dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl" + dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, dns-resolver, dns-zone, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl" fi if ! fn-in-array "$KEY" "${GLOBAL_KEYS[@]}" && [[ "$APP" == "--global" ]]; then diff --git a/plugins/nginx-vhosts/template_test.go b/plugins/nginx-vhosts/template_test.go index 4248e5532..ebfb02cea 100644 --- a/plugins/nginx-vhosts/template_test.go +++ b/plugins/nginx-vhosts/template_test.go @@ -30,6 +30,7 @@ func defaultVars() map[string]interface{} { "SSL_INUSE": "", "APP_SSL_PATH": "/home/dokku/app/tls", "DOKKU_APP_WEB_LISTENERS": "127.0.0.1:5000", + "DOKKU_APP_WEB_LISTENER_HOST": "app.web.docker", "PROXY_PORT_MAP": "http:80:5000", "PROXY_UPSTREAM_PORTS": "5000", "PROXY_PORT": "80", @@ -37,6 +38,8 @@ func defaultVars() map[string]interface{} { "PROXY_KEEPALIVE": "", "NGINX_BIND_ADDRESS_IP4": "", "NGINX_BIND_ADDRESS_IP6": "::", + "NGINX_DNS_RESOLVER": "127.0.0.1:1053", + "NGINX_DNS_ZONE": "docker", "NGINX_ACCESS_LOG_PATH": "/var/log/nginx/app-access.log", "NGINX_ACCESS_LOG_FORMAT": "", "NGINX_ERROR_LOG_PATH": "/var/log/nginx/app-error.log", @@ -95,7 +98,10 @@ func mustNotContain(t *testing.T, out, needle string) { func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { out := renderTemplate(t, defaultVars()) mustContain(t, out, "listen [::]:80;") - mustContain(t, out, "proxy_pass http://app-5000;") + mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;") + mustContain(t, out, `set $dokku_upstream "app.web.docker:5000";`) + mustContain(t, out, "proxy_pass http://$dokku_upstream;") + mustNotContain(t, out, "upstream app-5000 {") mustContain(t, out, "error_page 500 501 502 503") mustContain(t, out, "server_name app.example.com;") mustNotContain(t, out, "ssl_certificate") @@ -103,6 +109,17 @@ func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) { mustNotContain(t, out, "http2_push_preload") } +func TestTemplate_HTTPOnlyBasicProxyWithResolverOff(t *testing.T) { + v := defaultVars() + v["NGINX_DNS_RESOLVER"] = "off" + out := renderTemplate(t, v) + mustContain(t, out, "listen [::]:80;") + mustContain(t, out, "proxy_pass http://app-5000;") + mustContain(t, out, "upstream app-5000 {") + mustNotContain(t, out, "resolver 127.0.0.1:1053") + mustNotContain(t, out, "$dokku_upstream") +} + func TestTemplate_HTTPSEmitsPushPreloadWhenSupported(t *testing.T) { v := defaultVars() v["PROXY_PORT_MAP"] = "https:443:5000" @@ -128,6 +145,7 @@ func TestTemplate_HTTPRedirectsToHTTPSWhenSSLInUse(t *testing.T) { v["PROXY_PORT_MAP"] = "http:80:5000 https:443:5000" v["SSL_INUSE"] = "true" v["SSL_SERVER_NAME"] = "app.example.com" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) port80, _, ok := strings.Cut(out, "listen [::]:443") @@ -170,6 +188,7 @@ func TestTemplate_NoWebListenersReturns502(t *testing.T) { v["DOKKU_APP_WEB_LISTENERS"] = "" out := renderTemplate(t, v) mustContain(t, out, "return 502;") + mustNotContain(t, out, "proxy_pass http://$dokku_upstream;") mustNotContain(t, out, "proxy_pass http://app-5000;") } @@ -177,18 +196,30 @@ func TestTemplate_GRPCNoSSL(t *testing.T) { v := defaultVars() v["PROXY_PORT_MAP"] = "grpc:50051:50051" v["PROXY_UPSTREAM_PORTS"] = "50051" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "grpc_pass grpc://app-50051;") mustContain(t, out, "http2") mustNotContain(t, out, "ssl_certificate") } +func TestTemplate_GRPCNoSSLWithResolver(t *testing.T) { + v := defaultVars() + v["PROXY_PORT_MAP"] = "grpc:50051:50051" + v["PROXY_UPSTREAM_PORTS"] = "50051" + out := renderTemplate(t, v) + mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;") + mustContain(t, out, `set $dokku_upstream "app.web.docker:50051";`) + mustContain(t, out, "grpc_pass grpc://$dokku_upstream;") +} + func TestTemplate_GRPCS(t *testing.T) { v := defaultVars() v["PROXY_PORT_MAP"] = "grpcs:443:50051" v["PROXY_UPSTREAM_PORTS"] = "50051" v["SSL_INUSE"] = "true" v["SSL_SERVER_NAME"] = "app.example.com" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "ssl_certificate /home/dokku/app/tls/server.crt;") mustContain(t, out, "grpc_pass grpc://app-50051;") @@ -221,6 +252,7 @@ func TestTemplate_UpstreamBlock(t *testing.T) { v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001" v["PROXY_UPSTREAM_PORTS"] = "5000 5001" v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "upstream app-5000 {") mustContain(t, out, "upstream app-5001 {") @@ -230,13 +262,26 @@ func TestTemplate_UpstreamBlock(t *testing.T) { mustContain(t, out, "server 10.0.0.2:5001;") } +func TestTemplate_UpstreamBlockOmittedInResolverMode(t *testing.T) { + v := defaultVars() + v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001" + v["PROXY_UPSTREAM_PORTS"] = "5000 5001" + v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000" + out := renderTemplate(t, v) + mustNotContain(t, out, "upstream app-5000 {") + mustNotContain(t, out, "upstream app-5001 {") + mustNotContain(t, out, "server 10.0.0.1:5000;") +} + func TestTemplate_UpstreamWithKeepalive(t *testing.T) { v := defaultVars() v["PROXY_KEEPALIVE"] = "16" + v["NGINX_DNS_RESOLVER"] = "off" out := renderTemplate(t, v) mustContain(t, out, "keepalive 16;") v2 := defaultVars() + v2["NGINX_DNS_RESOLVER"] = "off" out2 := renderTemplate(t, v2) mustNotContain(t, out2, "keepalive 16;") } diff --git a/plugins/nginx-vhosts/templates/nginx.conf.sigil b/plugins/nginx-vhosts/templates/nginx.conf.sigil index e43760b6f..57e6ec3c4 100644 --- a/plugins/nginx-vhosts/templates/nginx.conf.sigil +++ b/plugins/nginx-vhosts/templates/nginx.conf.sigil @@ -84,7 +84,13 @@ server { gzip_vary on; gzip_comp_level 6; +{{ if ne $.NGINX_DNS_RESOLVER "off" }} + resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; + set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; + proxy_pass http://$dokku_upstream; +{{ else }} proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; +{{ end }} {{ if and $is_ssl (eq $.HTTP2_PUSH_SUPPORTED "true") }}http2_push_preload on; {{ end }} proxy_http_version {{ $.PROXY_HTTP_VERSION }}; proxy_connect_timeout {{ $.PROXY_CONNECT_TIMEOUT }}; @@ -166,7 +172,13 @@ server { {{ end }} location / { +{{ if ne $.NGINX_DNS_RESOLVER "off" }} + resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off; + set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}"; + grpc_pass grpc://$dokku_upstream; +{{ else }} grpc_pass grpc://{{ $.APP }}-{{ $upstream_port }}; +{{ end }} } client_max_body_size {{ $.CLIENT_MAX_BODY_SIZE }}; @@ -176,6 +188,7 @@ server { {{ end }} {{ end }} +{{ if eq $.NGINX_DNS_RESOLVER "off" }} {{ if $.DOKKU_APP_WEB_LISTENERS }} {{ range $upstream_port := $.PROXY_UPSTREAM_PORTS | split " " }} upstream {{ $.APP }}-{{ $upstream_port }} { @@ -185,7 +198,7 @@ upstream {{ $.APP }}-{{ $upstream_port }} { {{ $listener_ip := index $listener_list 0 }} server {{ $listener_ip }}:{{ $upstream_port }};{{ end }} } -{{ end }}{{ end }} +{{ end }}{{ end }}{{ end }} {{ range $entry := $.PROXY_ROUTE_LISTENERS | split " " }} {{ if $entry }} diff --git a/tests/unit/nginx-vhosts_17.bats b/tests/unit/nginx-vhosts_17.bats new file mode 100644 index 000000000..01c67e7c8 --- /dev/null +++ b/tests/unit/nginx-vhosts_17.bats @@ -0,0 +1,139 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup + [[ -f "$DOKKU_ROOT/VHOST" ]] && cp -fp "$DOKKU_ROOT/VHOST" "$DOKKU_ROOT/VHOST.bak" + create_app +} + +teardown() { + dokku --quiet nginx:set --global dns-resolver + dokku --quiet nginx:set --global dns-zone + destroy_app + [[ -f "$DOKKU_ROOT/VHOST.bak" ]] && mv "$DOKKU_ROOT/VHOST.bak" "$DOKKU_ROOT/VHOST" && chown dokku:dokku "$DOKKU_ROOT/VHOST" + global_teardown +} + +@test "(nginx-vhosts:set) dns-resolver and dns-zone defaults report 127.0.0.1:1053 and docker" { + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-resolver" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "127.0.0.1:1053" + + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-zone" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker" +} + +@test "(nginx-vhosts:set) dns-resolver and dns-zone can be overridden globally" { + run /bin/bash -c "dokku nginx:set --global dns-resolver 127.0.0.2:5353" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:set --global dns-zone test" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-resolver" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "127.0.0.2:5353" + + run /bin/bash -c "dokku nginx:report $TEST_APP --nginx-computed-dns-zone" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "test" +} + +@test "(nginx-vhosts) generated nginx.conf uses resolver and variable-based proxy_pass by default" { + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "resolver 127.0.0.1:1053 valid=10s ipv6=off" + assert_output_contains "set \$dokku_upstream" + assert_output_contains "${TEST_APP}.web.docker:" + assert_output_contains "proxy_pass http://\$dokku_upstream" + assert_output_not_contains "upstream ${TEST_APP}-" +} + +@test "(nginx-vhosts) generated nginx.conf falls back to IP-based upstream when dns-resolver=off" { + run /bin/bash -c "dokku nginx:set --global dns-resolver off" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_contains "resolver 127.0.0.1:1053" + assert_output_not_contains "\$dokku_upstream" + assert_output_contains "upstream ${TEST_APP}-" +} + +@test "(nginx-vhosts) setting dns-resolver to empty re-engages the default resolver path" { + run /bin/bash -c "dokku nginx:set --global dns-resolver off" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + assert_output_contains "upstream ${TEST_APP}-" + + run /bin/bash -c "dokku nginx:set --global dns-resolver" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku proxy:build-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "resolver 127.0.0.1:1053 valid=10s ipv6=off" + assert_output_not_contains "upstream ${TEST_APP}-" +} + +@test "(nginx-vhosts) vendored nginx.conf.sigil renders IP-based upstream regardless of dns-resolver" { + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP custom_nginx_template + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku nginx:show-config $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_contains "resolver 127.0.0.1:1053" + assert_output_not_contains "\$dokku_upstream" + assert_output_contains "upstream ${TEST_APP}-" +}