github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

feat: resolve nginx upstreams via coredns-docker

Browse Source 
Stops nginx from routing to stale container IPs after host reboots, Docker upgrades without live-restore, or runtime container restarts. The bundled `nginx.conf.sigil` can use an nginx `resolver` directive against the local `coredns-docker` socket and a variable-based `proxy_pass` keyed on `<app>.<process-type>.docker`, so the upstream is resolved at request time honoring DNS TTL. `coredns-docker` becomes a hard `Depends:` of the dokku deb and is bundled in the Docker image under runit. Closes #8313.
This commit is contained in:
Jose Diaz-Gonzalez
2026-11-06 20:43:16 -05:00
parent bcd563c2db
commit 5241fd4be2
16 changed files with 419 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
plugins/nginx-vhosts/functions
View File

@@ -324,6 +324,8 @@ fn-nginx-vhosts-render-template() {
local NGINX_BIND_ADDRESS_IP4="$(fn-nginx-computed-bind-address-ipv4 "$APP")"
local NGINX_BIND_ADDRESS_IP6="$(fn-nginx-computed-bind-address-ipv6 "$APP")"
local NGINX_DNS_RESOLVER="$(fn-nginx-computed-dns-resolver "$APP")"
local NGINX_DNS_ZONE="$(fn-nginx-computed-dns-zone "$APP")"
local NGINX_UNDERSCORE_IN_HEADERS="$(fn-nginx-computed-underscore-in-headers "$APP")"
local PROXY_X_FORWARDED_FOR="$(fn-nginx-computed-x-forwarded-for-value "$APP")"
local PROXY_X_FORWARDED_PORT="$(fn-nginx-computed-x-forwarded-port-value "$APP")"
@@ -354,6 +356,8 @@ fn-nginx-vhosts-render-template() {
NGINX_ERROR_LOG_PATH="$NGINX_ERROR_LOG_PATH"
NGINX_BIND_ADDRESS_IP4="$NGINX_BIND_ADDRESS_IP4"
NGINX_BIND_ADDRESS_IP6="$NGINX_BIND_ADDRESS_IP6"
NGINX_DNS_RESOLVER="$NGINX_DNS_RESOLVER"
NGINX_DNS_ZONE="$NGINX_DNS_ZONE"
HTTP2_DIRECTIVE_SUPPORTED="$HTTP2_DIRECTIVE_SUPPORTED"
HTTP2_PUSH_SUPPORTED="$HTTP2_PUSH_SUPPORTED"
GRPC_SUPPORTED="$GRPC_SUPPORTED"
@@ -386,21 +390,29 @@ fn-nginx-vhosts-render-template() {
PROXY_X_FORWARDED_SSL="$PROXY_X_FORWARDED_SSL")
if [[ "$IS_DEPLOYED_WITH_LISTENERS" == "true" ]]; then
local line PROC_TYPE LISTENERS UPP_PROC_TYPE
local line PROC_TYPE LISTENERS UPP_PROC_TYPE LISTENER_HOST
while read -r line || [[ -n "$line" ]]; do
PROC_TYPE=${line%%=*}
LISTENERS="$(plugn trigger network-get-listeners "$APP" "$PROC_TYPE" | xargs)"
UPP_PROC_TYPE="${PROC_TYPE^^}"
UPP_PROC_TYPE="${UPP_PROC_TYPE//-/_}"
SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENERS=$LISTENERS")
LISTENER_HOST=""
if [[ -n "$NGINX_DNS_ZONE" ]]; then
LISTENER_HOST="${APP}.${PROC_TYPE}.${NGINX_DNS_ZONE}"
fi
SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENER_HOST=$LISTENER_HOST")
done < <(plugn trigger ps-current-scale "$APP")
elif [[ "$PRE_VALIDATE" == "true" ]]; then
# During pre-validation no listeners exist yet; supply a placeholder so
# any upstream block in the template renders with a static server entry
# and nginx -t does not bail out on "host not found in upstream".
# During pre-validation no listeners exist yet; supply placeholders so
# both the legacy IP-based upstream block and the DNS-based resolver
# branch render syntactically valid nginx config that nginx -t can
# parse (no "host not found in upstream" and no empty set $var).
SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENERS="127.0.0.1:5000")
SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENER_HOST="127.0.0.1")
else
SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENERS="")
SIGIL_PARAMS+=(DOKKU_APP_WEB_LISTENER_HOST="")
fi
sigil "${SIGIL_PARAMS[@]}" | cat -s >"$OUTPUT_PATH"

42
plugins/nginx-vhosts/internal-functions
View File

@@ -172,6 +172,48 @@ fn-nginx-global-disable-custom-config() {
fn-get-property --app "$APP" --global "disable-custom-config"
}
fn-nginx-dns-resolver() {
declare desc="get the configured dns-resolver value"
declare APP="$1"
fn-get-property --app "$APP" "dns-resolver"
}
fn-nginx-computed-dns-resolver() {
declare desc="get the computed dns-resolver value"
declare APP="$1"
fn-get-property --app "$APP" --computed "dns-resolver"
}
fn-nginx-global-dns-resolver() {
declare desc="get the global dns-resolver value"
declare APP="$1"
fn-get-property --app "$APP" --global "dns-resolver"
}
fn-nginx-dns-zone() {
declare desc="get the configured dns-zone value"
declare APP="$1"
fn-get-property --app "$APP" "dns-zone"
}
fn-nginx-computed-dns-zone() {
declare desc="get the computed dns-zone value"
declare APP="$1"
fn-get-property --app "$APP" --computed "dns-zone"
}
fn-nginx-global-dns-zone() {
declare desc="get the global dns-zone value"
declare APP="$1"
fn-get-property --app "$APP" --global "dns-zone"
}
fn-nginx-error-log-path() {
declare desc="get the configured error log path"
declare APP="$1"

34
plugins/nginx-vhosts/nginx_vhosts.go
View File

@@ -150,6 +150,40 @@ func AppDisableCustomConfig(appName string) string {
return common.PropertyGet("nginx", appName, "disable-custom-config")
}
func AppDnsResolver(appName string) string {
return common.PropertyGet("nginx", appName, "dns-resolver")
}
func ComputedDnsResolver(appName string) string {
appValue := AppDnsResolver(appName)
if appValue != "" {
return appValue
}
return GlobalDnsResolver()
}
func GlobalDnsResolver() string {
return common.PropertyGetDefault("nginx", "--global", "dns-resolver", "127.0.0.1:1053")
}
func AppDnsZone(appName string) string {
return common.PropertyGet("nginx", appName, "dns-zone")
}
func ComputedDnsZone(appName string) string {
appValue := AppDnsZone(appName)
if appValue != "" {
return appValue
}
return GlobalDnsZone()
}
func GlobalDnsZone() string {
return common.PropertyGetDefault("nginx", "--global", "dns-zone", "docker")
}
func ComputedDisableCustomConfig(appName string) string {
appValue := AppDisableCustomConfig(appName)
if appValue != "" {

2
plugins/nginx-vhosts/report.go
View File

@@ -29,6 +29,8 @@ func nginxReportProps() []nginxReportProp {
{"client-header-timeout", AppClientHeaderTimeout, ComputedClientHeaderTimeout, wrapGlobal(GlobalClientHeaderTimeout)},
{"client-max-body-size", AppClientMaxBodySize, ComputedClientMaxBodySize, wrapGlobal(GlobalClientMaxBodySize)},
{"disable-custom-config", AppDisableCustomConfig, ComputedDisableCustomConfig, wrapGlobal(GlobalDisableCustomConfig)},
{"dns-resolver", AppDnsResolver, ComputedDnsResolver, wrapGlobal(GlobalDnsResolver)},
{"dns-zone", AppDnsZone, ComputedDnsZone, wrapGlobal(GlobalDnsZone)},
{"error-log-path", AppErrorLogPath, ComputedErrorLogPath, wrapGlobal(GlobalErrorLogPath)},
{"hsts-include-subdomains", AppHSTSIncludeSubdomains, ComputedHSTSIncludeSubdomains, wrapGlobal(GlobalHSTSIncludeSubdomains)},
{"hsts-max-age", AppHSTSMaxAge, ComputedHSTSMaxAge, wrapGlobal(GlobalHSTSMaxAge)},

12
plugins/nginx-vhosts/src/nginx-property/nginx-property.go
View File

@@ -51,6 +51,10 @@ func appValue(appName string, property string) string {
value = nginx_vhosts.AppClientHeaderTimeout(appName)
case "disable-custom-config":
value = nginx_vhosts.AppDisableCustomConfig(appName)
case "dns-resolver":
value = nginx_vhosts.AppDnsResolver(appName)
case "dns-zone":
value = nginx_vhosts.AppDnsZone(appName)
case "error-log-path":
value = nginx_vhosts.AppErrorLogPath(appName)
case "hsts-include-subdomains":
@@ -123,6 +127,10 @@ func computedValue(appName string, property string) string {
value = nginx_vhosts.ComputedClientMaxBodySize(appName)
case "disable-custom-config":
value = nginx_vhosts.ComputedDisableCustomConfig(appName)
case "dns-resolver":
value = nginx_vhosts.ComputedDnsResolver(appName)
case "dns-zone":
value = nginx_vhosts.ComputedDnsZone(appName)
case "error-log-path":
value = nginx_vhosts.ComputedErrorLogPath(appName)
case "hsts-include-subdomains":
@@ -195,6 +203,10 @@ func globalValue(appName string, property string) string {
value = nginx_vhosts.GlobalClientMaxBodySize()
case "disable-custom-config":
value = nginx_vhosts.GlobalDisableCustomConfig()
case "dns-resolver":
value = nginx_vhosts.GlobalDnsResolver()
case "dns-zone":
value = nginx_vhosts.GlobalDnsZone()
case "error-log-path":
value = nginx_vhosts.GlobalErrorLogPath()
case "hsts-include-subdomains":

6
plugins/nginx-vhosts/subcommands/set
View File

@@ -9,13 +9,13 @@ cmd-nginx-set() {
declare cmd="nginx:set"
[[ "$1" == "$cmd" ]] && shift 1
declare APP="$1" KEY="$2" VALUE="$3"
local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl")
local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl")
local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl")
local GLOBAL_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-body-timeout" "client-header-timeout" "client-max-body-size" "disable-custom-config" "dns-resolver" "dns-zone" "error-log-path" "hsts-include-subdomains" "hsts-max-age" "hsts-preload" "hsts" "keepalive-timeout" "lingering-timeout" "nginx-conf-sigil-path" "nginx-service-command" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "proxy-http-version" "proxy-connect-timeout" "proxy-read-timeout" "proxy-send-timeout" "proxy-keepalive" "send-timeout" "underscore-in-headers" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl")
[[ -z "$KEY" ]] && dokku_log_fail "No key specified"
if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then
dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl"
dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-body-timeout, client-header-timeout, client-max-body-size, disable-custom-config, dns-resolver, dns-zone, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, keepalive-timeout, lingering-timeout, nginx-conf-sigil-path, nginx-service-command, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, proxy-http-version, proxy-connect-timeout, proxy-read-timeout, proxy-send-timeout, proxy-keepalive, send-timeout, underscore-in-headers, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl"
fi
if ! fn-in-array "$KEY" "${GLOBAL_KEYS[@]}" && [[ "$APP" == "--global" ]]; then

47
plugins/nginx-vhosts/template_test.go
View File

@@ -30,6 +30,7 @@ func defaultVars() map[string]interface{} {
"SSL_INUSE": "",
"APP_SSL_PATH": "/home/dokku/app/tls",
"DOKKU_APP_WEB_LISTENERS": "127.0.0.1:5000",
"DOKKU_APP_WEB_LISTENER_HOST": "app.web.docker",
"PROXY_PORT_MAP": "http:80:5000",
"PROXY_UPSTREAM_PORTS": "5000",
"PROXY_PORT": "80",
@@ -37,6 +38,8 @@ func defaultVars() map[string]interface{} {
"PROXY_KEEPALIVE": "",
"NGINX_BIND_ADDRESS_IP4": "",
"NGINX_BIND_ADDRESS_IP6": "::",
"NGINX_DNS_RESOLVER": "127.0.0.1:1053",
"NGINX_DNS_ZONE": "docker",
"NGINX_ACCESS_LOG_PATH": "/var/log/nginx/app-access.log",
"NGINX_ACCESS_LOG_FORMAT": "",
"NGINX_ERROR_LOG_PATH": "/var/log/nginx/app-error.log",
@@ -95,7 +98,10 @@ func mustNotContain(t *testing.T, out, needle string) {
func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) {
out := renderTemplate(t, defaultVars())
mustContain(t, out, "listen [::]:80;")
mustContain(t, out, "proxy_pass http://app-5000;")
mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;")
mustContain(t, out, `set $dokku_upstream "app.web.docker:5000";`)
mustContain(t, out, "proxy_pass http://$dokku_upstream;")
mustNotContain(t, out, "upstream app-5000 {")
mustContain(t, out, "error_page 500 501 502 503")
mustContain(t, out, "server_name app.example.com;")
mustNotContain(t, out, "ssl_certificate")
@@ -103,6 +109,17 @@ func TestTemplate_HTTPOnlyBasicProxy(t *testing.T) {
mustNotContain(t, out, "http2_push_preload")
}
func TestTemplate_HTTPOnlyBasicProxyWithResolverOff(t *testing.T) {
v := defaultVars()
v["NGINX_DNS_RESOLVER"] = "off"
out := renderTemplate(t, v)
mustContain(t, out, "listen [::]:80;")
mustContain(t, out, "proxy_pass http://app-5000;")
mustContain(t, out, "upstream app-5000 {")
mustNotContain(t, out, "resolver 127.0.0.1:1053")
mustNotContain(t, out, "$dokku_upstream")
}
func TestTemplate_HTTPSEmitsPushPreloadWhenSupported(t *testing.T) {
v := defaultVars()
v["PROXY_PORT_MAP"] = "https:443:5000"
@@ -128,6 +145,7 @@ func TestTemplate_HTTPRedirectsToHTTPSWhenSSLInUse(t *testing.T) {
v["PROXY_PORT_MAP"] = "http:80:5000 https:443:5000"
v["SSL_INUSE"] = "true"
v["SSL_SERVER_NAME"] = "app.example.com"
v["NGINX_DNS_RESOLVER"] = "off"
out := renderTemplate(t, v)
port80, _, ok := strings.Cut(out, "listen [::]:443")
@@ -170,6 +188,7 @@ func TestTemplate_NoWebListenersReturns502(t *testing.T) {
v["DOKKU_APP_WEB_LISTENERS"] = ""
out := renderTemplate(t, v)
mustContain(t, out, "return 502;")
mustNotContain(t, out, "proxy_pass http://$dokku_upstream;")
mustNotContain(t, out, "proxy_pass http://app-5000;")
}
@@ -177,18 +196,30 @@ func TestTemplate_GRPCNoSSL(t *testing.T) {
v := defaultVars()
v["PROXY_PORT_MAP"] = "grpc:50051:50051"
v["PROXY_UPSTREAM_PORTS"] = "50051"
v["NGINX_DNS_RESOLVER"] = "off"
out := renderTemplate(t, v)
mustContain(t, out, "grpc_pass grpc://app-50051;")
mustContain(t, out, "http2")
mustNotContain(t, out, "ssl_certificate")
}
func TestTemplate_GRPCNoSSLWithResolver(t *testing.T) {
v := defaultVars()
v["PROXY_PORT_MAP"] = "grpc:50051:50051"
v["PROXY_UPSTREAM_PORTS"] = "50051"
out := renderTemplate(t, v)
mustContain(t, out, "resolver 127.0.0.1:1053 valid=10s ipv6=off;")
mustContain(t, out, `set $dokku_upstream "app.web.docker:50051";`)
mustContain(t, out, "grpc_pass grpc://$dokku_upstream;")
}
func TestTemplate_GRPCS(t *testing.T) {
v := defaultVars()
v["PROXY_PORT_MAP"] = "grpcs:443:50051"
v["PROXY_UPSTREAM_PORTS"] = "50051"
v["SSL_INUSE"] = "true"
v["SSL_SERVER_NAME"] = "app.example.com"
v["NGINX_DNS_RESOLVER"] = "off"
out := renderTemplate(t, v)
mustContain(t, out, "ssl_certificate /home/dokku/app/tls/server.crt;")
mustContain(t, out, "grpc_pass grpc://app-50051;")
@@ -221,6 +252,7 @@ func TestTemplate_UpstreamBlock(t *testing.T) {
v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001"
v["PROXY_UPSTREAM_PORTS"] = "5000 5001"
v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000"
v["NGINX_DNS_RESOLVER"] = "off"
out := renderTemplate(t, v)
mustContain(t, out, "upstream app-5000 {")
mustContain(t, out, "upstream app-5001 {")
@@ -230,13 +262,26 @@ func TestTemplate_UpstreamBlock(t *testing.T) {
mustContain(t, out, "server 10.0.0.2:5001;")
}
func TestTemplate_UpstreamBlockOmittedInResolverMode(t *testing.T) {
v := defaultVars()
v["PROXY_PORT_MAP"] = "http:80:5000 http:8080:5001"
v["PROXY_UPSTREAM_PORTS"] = "5000 5001"
v["DOKKU_APP_WEB_LISTENERS"] = "10.0.0.1:5000 10.0.0.2:5000"
out := renderTemplate(t, v)
mustNotContain(t, out, "upstream app-5000 {")
mustNotContain(t, out, "upstream app-5001 {")
mustNotContain(t, out, "server 10.0.0.1:5000;")
}
func TestTemplate_UpstreamWithKeepalive(t *testing.T) {
v := defaultVars()
v["PROXY_KEEPALIVE"] = "16"
v["NGINX_DNS_RESOLVER"] = "off"
out := renderTemplate(t, v)
mustContain(t, out, "keepalive 16;")
v2 := defaultVars()
v2["NGINX_DNS_RESOLVER"] = "off"
out2 := renderTemplate(t, v2)
mustNotContain(t, out2, "keepalive 16;")
}

15
plugins/nginx-vhosts/templates/nginx.conf.sigil
View File

@@ -84,7 +84,13 @@ server {
gzip_vary on;
gzip_comp_level 6;
{{ if ne $.NGINX_DNS_RESOLVER "off" }}
resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off;
set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}";
proxy_pass http://$dokku_upstream;
{{ else }}
proxy_pass http://{{ $.APP }}-{{ $upstream_port }};
{{ end }}
{{ if and $is_ssl (eq $.HTTP2_PUSH_SUPPORTED "true") }}http2_push_preload on; {{ end }}
proxy_http_version {{ $.PROXY_HTTP_VERSION }};
proxy_connect_timeout {{ $.PROXY_CONNECT_TIMEOUT }};
@@ -166,7 +172,13 @@ server {
{{ end }}
location / {
{{ if ne $.NGINX_DNS_RESOLVER "off" }}
resolver {{ $.NGINX_DNS_RESOLVER }} valid=10s ipv6=off;
set $dokku_upstream "{{ $.DOKKU_APP_WEB_LISTENER_HOST }}:{{ $upstream_port }}";
grpc_pass grpc://$dokku_upstream;
{{ else }}
grpc_pass grpc://{{ $.APP }}-{{ $upstream_port }};
{{ end }}
}
client_max_body_size {{ $.CLIENT_MAX_BODY_SIZE }};
@@ -176,6 +188,7 @@ server {
{{ end }}
{{ end }}
{{ if eq $.NGINX_DNS_RESOLVER "off" }}
{{ if $.DOKKU_APP_WEB_LISTENERS }}
{{ range $upstream_port := $.PROXY_UPSTREAM_PORTS | split " " }}
upstream {{ $.APP }}-{{ $upstream_port }} {
@@ -185,7 +198,7 @@ upstream {{ $.APP }}-{{ $upstream_port }} {
{{ $listener_ip := index $listener_list 0 }}
server {{ $listener_ip }}:{{ $upstream_port }};{{ end }}
}
{{ end }}{{ end }}
{{ end }}{{ end }}{{ end }}
{{ range $entry := $.PROXY_ROUTE_LISTENERS | split " " }}
{{ if $entry }}