diff --git a/plugins/scheduler-k3s/ingress_route_template_test.go b/plugins/scheduler-k3s/ingress_route_template_test.go new file mode 100644 index 000000000..ec3493bb5 --- /dev/null +++ b/plugins/scheduler-k3s/ingress_route_template_test.go @@ -0,0 +1,296 @@ +package scheduler_k3s + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/engine" +) + +func renderIngressRouteTemplate(t *testing.T, values map[string]interface{}) []map[string]interface{} { + t.Helper() + + chartDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n") + if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil { + t.Fatalf("write Chart.yaml: %v", err) + } + + ingressRouteTpl, err := templates.ReadFile("templates/chart/ingress-route.yaml") + if err != nil { + t.Fatalf("read ingress-route template: %v", err) + } + if err := os.WriteFile(filepath.Join(chartDir, "templates", "ingress-route.yaml"), ingressRouteTpl, 0o644); err != nil { + t.Fatalf("write ingress-route template: %v", err) + } + + helpersTpl, err := templates.ReadFile("templates/chart/_helpers.tpl") + if err != nil { + t.Fatalf("read _helpers: %v", err) + } + if err := os.WriteFile(filepath.Join(chartDir, "templates", "_helpers.tpl"), helpersTpl, 0o644); err != nil { + t.Fatalf("write _helpers: %v", err) + } + + loaded, err := loader.Load(chartDir) + if err != nil { + t.Fatalf("load chart: %v", err) + } + + renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil) + if err != nil { + t.Fatalf("ToRenderValues: %v", err) + } + + rendered, err := engine.Render(loaded, renderValues) + if err != nil { + t.Fatalf("render: %v", err) + } + + manifest := rendered["test/templates/ingress-route.yaml"] + + var docs []map[string]interface{} + decoder := yaml.NewDecoder(strings.NewReader(manifest)) + for { + var doc map[string]interface{} + if err := decoder.Decode(&doc); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("yaml decode failed: %v\nrendered:\n%s", err, manifest) + } + if doc != nil { + docs = append(docs, doc) + } + } + + return docs +} + +func testIngressRouteValues(tlsEnabled bool) map[string]interface{} { + web := map[string]interface{}{ + "domains": []interface{}{ + map[string]interface{}{"name": "app.example.com"}, + }, + "port_maps": []interface{}{ + map[string]interface{}{ + "name": "http-80-5000", + "scheme": "http", + "container_port": 5000, + }, + }, + "tls": map[string]interface{}{ + "enabled": tlsEnabled, + "use_imported_cert": false, + }, + } + + return map[string]interface{}{ + "global": map[string]interface{}{ + "app_name": "myapp", + "namespace": "myns", + "network": map[string]interface{}{ + "ingress_class": "traefik", + }, + }, + "processes": map[string]interface{}{ + "web": map[string]interface{}{ + "web": web, + }, + }, + } +} + +func findDocByName(t *testing.T, docs []map[string]interface{}, name string) map[string]interface{} { + t.Helper() + for _, doc := range docs { + metadata, _ := doc["metadata"].(map[string]interface{}) + if metadata["name"] == name { + return doc + } + } + t.Fatalf("document %q not found in %#v", name, docs) + return nil +} + +func TestIngressRouteTemplateTLSCreatesSeparateHTTPAndHTTPSRoutes(t *testing.T) { + docs := renderIngressRouteTemplate(t, testIngressRouteValues(true)) + if len(docs) != 2 { + t.Fatalf("expected 2 ingress routes when tls enabled, got %d", len(docs)) + } + + httpRoute := findDocByName(t, docs, "myapp-web-http-80-5000") + httpSpec := httpRoute["spec"].(map[string]interface{}) + if got := httpSpec["entryPoints"]; len(got.([]interface{})) != 1 || got.([]interface{})[0] != "web" { + t.Fatalf("expected HTTP route entryPoints [web], got %#v", got) + } + httpRoutes := httpSpec["routes"].([]interface{}) + httpRoute0 := httpRoutes[0].(map[string]interface{}) + if _, ok := httpRoute0["middlewares"]; !ok { + t.Fatalf("expected HTTP route to contain redirect middleware, got %#v", httpRoute0) + } + if _, ok := httpSpec["tls"]; ok { + t.Fatalf("expected HTTP route to omit tls block, got %#v", httpSpec["tls"]) + } + + httpsRoute := findDocByName(t, docs, "myapp-web-http-80-5000-websecure") + httpsSpec := httpsRoute["spec"].(map[string]interface{}) + if got := httpsSpec["entryPoints"]; len(got.([]interface{})) != 1 || got.([]interface{})[0] != "websecure" { + t.Fatalf("expected HTTPS route entryPoints [websecure], got %#v", got) + } + httpsRoutes := httpsSpec["routes"].([]interface{}) + httpsRoute0 := httpsRoutes[0].(map[string]interface{}) + if _, ok := httpsRoute0["middlewares"]; ok { + t.Fatalf("expected HTTPS route to omit redirect middleware, got %#v", httpsRoute0["middlewares"]) + } + tls, ok := httpsSpec["tls"].(map[string]interface{}) + if !ok { + t.Fatalf("expected HTTPS route to include tls block, got %#v", httpsSpec["tls"]) + } + if tls["secretName"] != "tls-myapp-web" { + t.Fatalf("expected HTTPS route secretName %q, got %#v", "tls-myapp-web", tls["secretName"]) + } +} + +func TestIngressRouteTemplateWithoutTLSKeepsSingleHTTPRoute(t *testing.T) { + docs := renderIngressRouteTemplate(t, testIngressRouteValues(false)) + if len(docs) != 1 { + t.Fatalf("expected 1 ingress route when tls disabled, got %d", len(docs)) + } + + httpRoute := findDocByName(t, docs, "myapp-web-http-80-5000") + httpSpec := httpRoute["spec"].(map[string]interface{}) + if got := httpSpec["entryPoints"]; len(got.([]interface{})) != 1 || got.([]interface{})[0] != "web" { + t.Fatalf("expected HTTP route entryPoints [web], got %#v", got) + } + httpRoutes := httpSpec["routes"].([]interface{}) + httpRoute0 := httpRoutes[0].(map[string]interface{}) + if _, ok := httpRoute0["middlewares"]; ok { + t.Fatalf("expected non-TLS route to omit redirect middleware, got %#v", httpRoute0["middlewares"]) + } + if _, ok := httpSpec["tls"]; ok { + t.Fatalf("expected non-TLS route to omit tls block, got %#v", httpSpec["tls"]) + } +} + +func TestIngressRouteTemplateTLSImportedCertUsesSharedSecretName(t *testing.T) { + values := testIngressRouteValues(true) + web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{}) + web["tls"].(map[string]interface{})["use_imported_cert"] = true + + docs := renderIngressRouteTemplate(t, values) + if len(docs) != 2 { + t.Fatalf("expected 2 ingress routes when tls enabled, got %d", len(docs)) + } + + httpsRoute := findDocByName(t, docs, "myapp-web-http-80-5000-websecure") + httpsSpec := httpsRoute["spec"].(map[string]interface{}) + tls, ok := httpsSpec["tls"].(map[string]interface{}) + if !ok { + t.Fatalf("expected HTTPS route to include tls block, got %#v", httpsSpec["tls"]) + } + if tls["secretName"] != "tls-myapp" { + t.Fatalf("expected HTTPS route secretName %q (imported cert), got %#v", "tls-myapp", tls["secretName"]) + } +} + +func TestIngressRouteTemplateMultipleDomainsRenderOneRoutePerDomain(t *testing.T) { + values := testIngressRouteValues(true) + web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{}) + web["domains"] = []interface{}{ + map[string]interface{}{"name": "app.example.com"}, + map[string]interface{}{"name": "www.example.com"}, + } + + docs := renderIngressRouteTemplate(t, values) + if len(docs) != 2 { + t.Fatalf("expected 2 ingress routes (one http, one websecure), got %d", len(docs)) + } + + httpRoute := findDocByName(t, docs, "myapp-web-http-80-5000") + httpRoutes := httpRoute["spec"].(map[string]interface{})["routes"].([]interface{}) + if len(httpRoutes) != 2 { + t.Fatalf("expected HTTP route to have 2 entries (one per domain), got %d", len(httpRoutes)) + } + for i, r := range httpRoutes { + route := r.(map[string]interface{}) + if _, ok := route["middlewares"]; !ok { + t.Fatalf("expected HTTP route entry %d to contain redirect middleware, got %#v", i, route) + } + } + if got := httpRoutes[0].(map[string]interface{})["match"]; got != "Host(`app.example.com`)" { + t.Fatalf("expected HTTP route[0] match Host(`app.example.com`), got %#v", got) + } + if got := httpRoutes[1].(map[string]interface{})["match"]; got != "Host(`www.example.com`)" { + t.Fatalf("expected HTTP route[1] match Host(`www.example.com`), got %#v", got) + } + + httpsRoute := findDocByName(t, docs, "myapp-web-http-80-5000-websecure") + httpsRoutes := httpsRoute["spec"].(map[string]interface{})["routes"].([]interface{}) + if len(httpsRoutes) != 2 { + t.Fatalf("expected HTTPS route to have 2 entries (one per domain), got %d", len(httpsRoutes)) + } + for i, r := range httpsRoutes { + route := r.(map[string]interface{}) + if _, ok := route["middlewares"]; ok { + t.Fatalf("expected HTTPS route entry %d to omit redirect middleware, got %#v", i, route) + } + } + if got := httpsRoutes[0].(map[string]interface{})["match"]; got != "Host(`app.example.com`)" { + t.Fatalf("expected HTTPS route[0] match Host(`app.example.com`), got %#v", got) + } + if got := httpsRoutes[1].(map[string]interface{})["match"]; got != "Host(`www.example.com`)" { + t.Fatalf("expected HTTPS route[1] match Host(`www.example.com`), got %#v", got) + } +} + +func TestIngressRouteTemplateNonTraefikIngressClassRendersNothing(t *testing.T) { + values := testIngressRouteValues(true) + values["global"].(map[string]interface{})["network"].(map[string]interface{})["ingress_class"] = "nginx" + + docs := renderIngressRouteTemplate(t, values) + if len(docs) != 0 { + t.Fatalf("expected 0 ingress routes when ingress_class is not traefik, got %d: %#v", len(docs), docs) + } +} + +func TestIngressRouteTemplateHTTPSPortMapSkippedWhenHTTPCovers(t *testing.T) { + values := testIngressRouteValues(true) + web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{}) + web["port_maps"] = []interface{}{ + map[string]interface{}{ + "name": "http-80-5000", + "scheme": "http", + "container_port": float64(5000), + }, + map[string]interface{}{ + "name": "https-443-5000", + "scheme": "https", + "container_port": float64(5000), + }, + } + + docs := renderIngressRouteTemplate(t, values) + if len(docs) != 2 { + t.Fatalf("expected 2 ingress routes (http + websecure for http-80-5000 only), got %d", len(docs)) + } + for _, doc := range docs { + name := doc["metadata"].(map[string]interface{})["name"].(string) + if strings.Contains(name, "https-443-5000") { + t.Fatalf("expected no IngressRoute for https-443-5000 (covered by http-80-5000), got %q", name) + } + } + findDocByName(t, docs, "myapp-web-http-80-5000") + findDocByName(t, docs, "myapp-web-http-80-5000-websecure") +} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index 24dd8e2ef..0d9db860a 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -13,6 +13,10 @@ {{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }} {{- continue }} {{- end }} +{{- range $rdx, $entryPoint := list "web" "websecure" }} +{{- if and (eq $entryPoint "websecure") (not $config.web.tls.enabled) }} +{{- continue }} +{{- end }} --- apiVersion: traefik.io/v1alpha1 kind: IngressRoute @@ -27,19 +31,16 @@ metadata: app.kubernetes.io/part-of: {{ $.Values.global.app_name }} {{ include "print.labels" (dict "config" $.Values.global "key" "traefik_ingressroute") | indent 4 }} {{ include "print.labels" (dict "config" $config "key" "traefik_ingressroute") | indent 4 }} - name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }} + name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }}{{- if eq $entryPoint "websecure" }}-{{ $entryPoint }}{{- end }} namespace: {{ $.Values.global.namespace }} spec: entryPoints: - {{- if $config.web.tls.enabled }} - - websecure - {{- end }} - - web + - {{ $entryPoint }} routes: {{- range $ddx, $domain := $config.web.domains }} - kind: Rule match: Host(`{{ $domain.name }}`) - {{- if $config.web.tls.enabled }} + {{- if and (eq $entryPoint "web") $config.web.tls.enabled }} middlewares: - name: {{ $.Values.global.app_name}}-{{ $processName }}-redirect-to-https namespace: {{ $.Values.global.namespace }} @@ -51,7 +52,7 @@ spec: port: {{ $port_map.name }} scheme: http {{- end }} - {{- if $config.web.tls.enabled }} + {{- if eq $entryPoint "websecure" }} tls: {{- if $config.web.tls.use_imported_cert }} secretName: tls-{{ $.Values.global.app_name }} @@ -62,3 +63,4 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/tests/unit/scheduler-k3s-certs-redirect.bats b/tests/unit/scheduler-k3s-certs-redirect.bats new file mode 100644 index 000000000..a3cadbacf --- /dev/null +++ b/tests/unit/scheduler-k3s-certs-redirect.bats @@ -0,0 +1,86 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup_local_tls() { + TLS=$BATS_TMPDIR/tls + mkdir -p $TLS + tar xf $BATS_TEST_DIRNAME/server_ssl.tar -C $TLS + sudo chown -R dokku:dokku $TLS +} + +teardown_local_tls() { + TLS=$BATS_TMPDIR/tls + rm -R $TLS +} + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" + setup_local_tls +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true + teardown_local_tls +} + +@test "(scheduler-k3s) [ingress] traefik redirects http to https when tls is enabled" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=traefik install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get ingressroute ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].middlewares[0].name}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "${TEST_APP}-web-redirect-to-https" + + run /bin/bash -c "kubectl get ingressroute ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.tls.secretName}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" + + run /bin/bash -c "kubectl get ingressroute ${TEST_APP}-web-http-80-5000-websecure -n default -o jsonpath='{.spec.tls.secretName}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "tls-${TEST_APP}" + + assert_http_redirect "http://$TEST_APP.dokku.me" "https://$TEST_APP.dokku.me/" + assert_http_localhost_response "https" "$TEST_APP.dokku.me" "443" "" "python/http.server" +}