Merge pull request #8911 from dokku/8910-k3s-scheduler-traefik-ingress-cannot-route-wildcard-domains

Route wildcard domains through traefik on k3s

docs/deployment/schedulers/k3s.md

@@ -343,6 +343,20 @@ In the above example, the `internal-web` process is exposed as a service. The `P
 > [!NOTE]
 > It is not possible to modify the port mapping, nor is it possible to assign domains or SSL to a non-web process.
 
+### Wildcard domains
+
+Both the `nginx` and `traefik` ingress classes route wildcard domains. Add the wildcard as a domain on the app:
+
+```shell
+dokku domains:add node-js-app '*.node-js-app.com'
+```
+
+A wildcard matches exactly one label, matching DNS itself, so `*.node-js-app.com` covers `api.node-js-app.com` but not `node-js-app.com` or `api.staging.node-js-app.com`. Add the apex as a separate domain if it should also be served.
+
+An exact domain always takes precedence over a wildcard, including across apps. If one app serves `*.node-js-app.com` and another serves `api.node-js-app.com`, requests for `api.node-js-app.com` are routed to the second app.
+
+Only the leading label may be wildcarded. A domain such as `api.*.node-js-app.com` is treated as a literal hostname and will not match anything.
+
 ### SSL Certificates
 
 #### Enabling letsencrypt integration
@@ -485,10 +499,7 @@ A `dns01` issuer can issue wildcard certificates. Add the wildcard as a domain o
 dokku domains:add node-js-app '*.node-js-app.com'
 ```
 
-> [!WARNING]
-> Wildcard domains require the `nginx` ingress class. The Traefik integration matches hosts exactly and will not route requests for a wildcard domain.
-
-Note that a Kubernetes wildcard host matches exactly one label, so `*.node-js-app.com` does not cover `node-js-app.com`. Add both domains if the apex should also be served.
+See [wildcard domains](#wildcard-domains) for how a wildcard is matched against incoming requests.
 
 #### Using imported SSL certificates
 

plugins/scheduler-k3s/ingress_route_template_test.go

@@ -281,6 +281,83 @@ func TestIngressRouteTemplateMultipleDomainsRenderOneRoutePerDomain(t *testing.T
 	}
 }
 
+func TestIngressRouteTemplateWildcardDomainUsesHostRegexp(t *testing.T) {
+	values := testIngressRouteValues(true)
+	web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{})
+	web["domains"] = []interface{}{
+		map[string]interface{}{"name": "*.example.com"},
+	}
+
+	docs := renderIngressRouteTemplate(t, values)
+	if len(docs) != 2 {
+		t.Fatalf("expected 2 ingress routes when tls enabled, got %d", len(docs))
+	}
+
+	for _, name := range []string{"myapp-web-http-80-5000", "myapp-web-http-80-5000-websecure"} {
+		routes := findDocByName(t, docs, name)["spec"].(map[string]interface{})["routes"].([]interface{})
+		if len(routes) != 1 {
+			t.Fatalf("expected %s to have 1 route, got %d", name, len(routes))
+		}
+
+		route := routes[0].(map[string]interface{})
+		if got := route["match"]; got != "HostRegexp(`{subdomain:[^.]+}.example.com`)" {
+			t.Fatalf("expected %s match HostRegexp(`{subdomain:[^.]+}.example.com`), got %#v", name, got)
+		}
+		if got := route["priority"]; got != 1 {
+			t.Fatalf("expected %s priority 1, got %#v", name, got)
+		}
+	}
+}
+
+func TestIngressRouteTemplateWildcardRouteYieldsToExactDomain(t *testing.T) {
+	values := testIngressRouteValues(false)
+	web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{})
+	web["domains"] = []interface{}{
+		map[string]interface{}{"name": "*.example.com"},
+		map[string]interface{}{"name": "app.example.com"},
+	}
+
+	docs := renderIngressRouteTemplate(t, values)
+	routes := findDocByName(t, docs, "myapp-web-http-80-5000")["spec"].(map[string]interface{})["routes"].([]interface{})
+	if len(routes) != 2 {
+		t.Fatalf("expected 2 routes (one per domain), got %d", len(routes))
+	}
+
+	wildcard := routes[0].(map[string]interface{})
+	if got := wildcard["match"]; got != "HostRegexp(`{subdomain:[^.]+}.example.com`)" {
+		t.Fatalf("expected wildcard match HostRegexp(`{subdomain:[^.]+}.example.com`), got %#v", got)
+	}
+	if got := wildcard["priority"]; got != 1 {
+		t.Fatalf("expected wildcard priority 1, got %#v", got)
+	}
+
+	exact := routes[1].(map[string]interface{})
+	if got := exact["match"]; got != "Host(`app.example.com`)" {
+		t.Fatalf("expected exact match Host(`app.example.com`), got %#v", got)
+	}
+	if got, ok := exact["priority"]; ok {
+		t.Fatalf("expected exact route to omit priority so traefik defaults it above the wildcard, got %#v", got)
+	}
+}
+
+func TestIngressRouteTemplateNonLeadingWildcardStaysExactHost(t *testing.T) {
+	values := testIngressRouteValues(false)
+	web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{})
+	web["domains"] = []interface{}{
+		map[string]interface{}{"name": "app.*.example.com"},
+	}
+
+	docs := renderIngressRouteTemplate(t, values)
+	routes := findDocByName(t, docs, "myapp-web-http-80-5000")["spec"].(map[string]interface{})["routes"].([]interface{})
+	route := routes[0].(map[string]interface{})
+	if got := route["match"]; got != "Host(`app.*.example.com`)" {
+		t.Fatalf("expected non-leading wildcard to stay Host(`app.*.example.com`), got %#v", got)
+	}
+	if got, ok := route["priority"]; ok {
+		t.Fatalf("expected non-leading wildcard to omit priority, got %#v", got)
+	}
+}
+
 func TestIngressRouteTemplateNonTraefikIngressClassRendersNothing(t *testing.T) {
 	values := testIngressRouteValues(true)
 	values["global"].(map[string]interface{})["network"].(map[string]interface{})["ingress_class"] = "nginx"

plugins/scheduler-k3s/ingress_template_test.go

@@ -0,0 +1,205 @@
+package scheduler_k3s
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+)
+
+func renderIngressTemplate(t *testing.T, values map[string]interface{}) []map[string]interface{} {
+	t.Helper()
+
+	chartDir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n")
+	if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil {
+		t.Fatalf("write Chart.yaml: %v", err)
+	}
+
+	ingressTpl, err := templates.ReadFile("templates/chart/ingress.yaml")
+	if err != nil {
+		t.Fatalf("read ingress template: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(chartDir, "templates", "ingress.yaml"), ingressTpl, 0o644); err != nil {
+		t.Fatalf("write ingress template: %v", err)
+	}
+
+	helpersTpl, err := templates.ReadFile("templates/chart/_helpers.tpl")
+	if err != nil {
+		t.Fatalf("read _helpers: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(chartDir, "templates", "_helpers.tpl"), helpersTpl, 0o644); err != nil {
+		t.Fatalf("write _helpers: %v", err)
+	}
+
+	loaded, err := loader.Load(chartDir)
+	if err != nil {
+		t.Fatalf("load chart: %v", err)
+	}
+
+	renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil)
+	if err != nil {
+		t.Fatalf("ToRenderValues: %v", err)
+	}
+
+	rendered, err := engine.Render(loaded, renderValues)
+	if err != nil {
+		t.Fatalf("render: %v", err)
+	}
+
+	manifest := rendered["test/templates/ingress.yaml"]
+
+	var docs []map[string]interface{}
+	decoder := yaml.NewDecoder(strings.NewReader(manifest))
+	for {
+		var doc map[string]interface{}
+		if err := decoder.Decode(&doc); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			t.Fatalf("yaml decode failed: %v\nrendered:\n%s", err, manifest)
+		}
+		if doc != nil {
+			docs = append(docs, doc)
+		}
+	}
+
+	return docs
+}
+
+func testIngressValues(tlsEnabled bool, domains ...map[string]interface{}) map[string]interface{} {
+	domainValues := []interface{}{}
+	for _, domain := range domains {
+		domainValues = append(domainValues, domain)
+	}
+
+	web := map[string]interface{}{
+		"domains": domainValues,
+		"port_maps": []interface{}{
+			map[string]interface{}{
+				"name":           "http-80-5000",
+				"scheme":         "http",
+				"container_port": 5000,
+			},
+		},
+		"tls": map[string]interface{}{
+			"enabled":           tlsEnabled,
+			"use_imported_cert": false,
+		},
+	}
+
+	return map[string]interface{}{
+		"global": map[string]interface{}{
+			"app_name":  "myapp",
+			"namespace": "myns",
+			"network": map[string]interface{}{
+				"ingress_class": "nginx",
+			},
+		},
+		"processes": map[string]interface{}{
+			"web": map[string]interface{}{
+				"web": web,
+			},
+		},
+	}
+}
+
+func ingressHost(t *testing.T, doc map[string]interface{}) string {
+	t.Helper()
+
+	rules, ok := doc["spec"].(map[string]interface{})["rules"].([]interface{})
+	if !ok || len(rules) != 1 {
+		t.Fatalf("expected ingress to contain a single rule, got %#v", doc["spec"])
+	}
+
+	host, ok := rules[0].(map[string]interface{})["host"].(string)
+	if !ok {
+		t.Fatalf("expected rule to contain a string host, got %#v", rules[0])
+	}
+
+	return host
+}
+
+func TestIngressTemplateExactDomainUsesSlugForObjectName(t *testing.T) {
+	values := testIngressValues(false, map[string]interface{}{"name": "app.example.com", "slug": "app-example-com"})
+
+	docs := renderIngressTemplate(t, values)
+	if len(docs) != 1 {
+		t.Fatalf("expected 1 ingress, got %d", len(docs))
+	}
+
+	doc := findDocByName(t, docs, "myapp-web-app-example-com")
+	if got := ingressHost(t, doc); got != "app.example.com" {
+		t.Fatalf("expected host app.example.com, got %q", got)
+	}
+}
+
+func TestIngressTemplateWildcardDomainRendersWildcardHost(t *testing.T) {
+	values := testIngressValues(true, map[string]interface{}{"name": "*.example.com", "slug": "wildcard-example-com"})
+
+	docs := renderIngressTemplate(t, values)
+	if len(docs) != 1 {
+		t.Fatalf("expected 1 ingress, got %d", len(docs))
+	}
+
+	doc := findDocByName(t, docs, "myapp-web-wildcard-example-com")
+	if got := ingressHost(t, doc); got != "*.example.com" {
+		t.Fatalf("expected host *.example.com, got %q", got)
+	}
+
+	tls, ok := doc["spec"].(map[string]interface{})["tls"].([]interface{})
+	if !ok || len(tls) != 1 {
+		t.Fatalf("expected ingress to contain a single tls entry, got %#v", doc["spec"])
+	}
+
+	hosts, ok := tls[0].(map[string]interface{})["hosts"].([]interface{})
+	if !ok || len(hosts) != 1 || hosts[0] != "*.example.com" {
+		t.Fatalf("expected tls hosts [*.example.com], got %#v", tls[0])
+	}
+	if got := tls[0].(map[string]interface{})["secretName"]; got != "tls-myapp-web" {
+		t.Fatalf("expected tls secretName tls-myapp-web, got %#v", got)
+	}
+}
+
+func TestIngressTemplateWildcardAndApexRenderDistinctObjects(t *testing.T) {
+	values := testIngressValues(false,
+		map[string]interface{}{"name": "*.example.com", "slug": "wildcard-example-com"},
+		map[string]interface{}{"name": "example.com", "slug": "example-com"},
+	)
+
+	docs := renderIngressTemplate(t, values)
+	if len(docs) != 2 {
+		t.Fatalf("expected 2 ingresses (one per domain), got %d", len(docs))
+	}
+
+	wildcard := findDocByName(t, docs, "myapp-web-wildcard-example-com")
+	if got := ingressHost(t, wildcard); got != "*.example.com" {
+		t.Fatalf("expected wildcard host *.example.com, got %q", got)
+	}
+
+	apex := findDocByName(t, docs, "myapp-web-example-com")
+	if got := ingressHost(t, apex); got != "example.com" {
+		t.Fatalf("expected apex host example.com, got %q", got)
+	}
+}
+
+func TestIngressTemplateNonNginxIngressClassRendersNothing(t *testing.T) {
+	values := testIngressValues(false, map[string]interface{}{"name": "app.example.com", "slug": "app-example-com"})
+	values["global"].(map[string]interface{})["network"].(map[string]interface{})["ingress_class"] = "traefik"
+
+	docs := renderIngressTemplate(t, values)
+	if len(docs) != 0 {
+		t.Fatalf("expected 0 ingresses when ingress_class is not nginx, got %d: %#v", len(docs), docs)
+	}
+}

plugins/scheduler-k3s/templates/chart/_helpers.tpl

@@ -16,6 +16,20 @@
 {{- end }}
 {{- end }}
 
+{{/*
+traefik.host.match renders a domain as a Traefik router rule. Traefik v2 resolves HostRegexp
+through gorilla/mux, where the pattern is a {name:regexp} template and every literal segment is
+quoted, so a wildcard label is spelled out as a named group rather than a raw regexp. The group
+matches a single label, mirroring both DNS and Kubernetes Ingress wildcard hosts.
+*/}}
+{{- define "traefik.host.match" -}}
+{{- if hasPrefix "*." . -}}
+HostRegexp(`{subdomain:[^.]+}.{{ trimPrefix "*." . }}`)
+{{- else -}}
+Host(`{{ . }}`)
+{{- end -}}
+{{- end -}}
+
 {{- define "primary.port" -}}
 {{- $found := dict -}}
 {{- range $idx, $port_map := . -}}

plugins/scheduler-k3s/templates/chart/ingress-route.yaml

@@ -39,7 +39,10 @@ spec:
   routes:
     {{- range $ddx, $domain := $config.web.domains }}
     - kind: Rule
-      match: Host(`{{ $domain.name }}`)
+      match: {{ include "traefik.host.match" $domain.name }}
+      {{- if hasPrefix "*." $domain.name }}
+      priority: 1
+      {{- end }}
       middlewares:
         - name: {{ $.Values.global.app_name}}-{{ $processName }}-compression
           namespace: {{ $.Values.global.namespace }}

tests/unit/scheduler-k3s-wildcard-1.bats

@@ -0,0 +1,93 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+EXACT_APP="rdmtestapp2"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s) [ingress] traefik routes wildcard domains and prefers exact domains" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=traefik install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP '*.dokku.me'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP HELLO=wildcard"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku apps:create $EXACT_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $EXACT_APP exact.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $EXACT_APP HELLO=exact"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$EXACT_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 30"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].match}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output 'HostRegexp(`{subdomain:[^.]+}.dokku.me`)'
+
+  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].priority}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "1"
+
+  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${EXACT_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].priority}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  assert_http_localhost_response "http" "anything.dokku.me" "80" "/hello" "wildcard"
+  assert_http_localhost_response "http" "exact.dokku.me" "80" "/hello" "exact"
+}

tests/unit/scheduler-k3s-wildcard-2.bats

@@ -0,0 +1,80 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup_wildcard_tls() {
+  TLS=$BATS_TMPDIR/tls
+  mkdir -p $TLS
+  tar xf $BATS_TEST_DIRNAME/server_ssl_wildcard.tar -C $TLS
+  sudo chown -R dokku:dokku $TLS
+}
+
+teardown_wildcard_tls() {
+  TLS=$BATS_TMPDIR/tls
+  rm -R $TLS
+}
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+  setup_wildcard_tls
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+  teardown_wildcard_tls
+}
+
+@test "(scheduler-k3s) [ingress] traefik serves a wildcard domain over https" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=traefik install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP '*.dokku.me'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 30"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000-websecure -n default -o jsonpath='{.spec.routes[0].match}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output 'HostRegexp(`{subdomain:[^.]+}.dokku.me`)'
+
+  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000-websecure -n default -o jsonpath='{.spec.tls.secretName}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "tls-${TEST_APP}"
+
+  assert_http_redirect "http://wild.dokku.me" "https://wild.dokku.me/"
+  assert_http_localhost_response "https" "wild.dokku.me" "443" "" "python/http.server"
+}