Merge pull request #8692 from Mordred/feat/scheduler-k3s-compression

Enable compression middleware for traefik when using scheduler-k3s

docs/getting-started/install/vagrant.md

@@ -21,10 +21,10 @@
     # - `DOKKU_IP`
     # - `FORWARDED_PORT`.
     cd path/to/dokku
-    
+
     # for most users
     vagrant up
-    
+
     # windows users must instead use the following in an elevated command prompt
     vagrant up dokku-windows
     ```
@@ -42,7 +42,7 @@
 
     ```shell
     # usually your key is already available under the current user's `~/.ssh/authorized_keys` file
-    cat ~/.ssh/authorized_keys | dokku ssh-keys:add admin
+    cat ~/.ssh/authorized_keys | sudo dokku ssh-keys:add admin
 
     # you can use any domain you already have access to
     dokku domains:set-global dokku.me

plugins/scheduler-k3s/ingress_route_template_test.go

@@ -124,6 +124,32 @@ func findDocByName(t *testing.T, docs []map[string]interface{}, name string) map
 	return nil
 }
 
+func middlewareNames(t *testing.T, route map[string]interface{}) []string {
+	t.Helper()
+
+	middlewares, ok := route["middlewares"].([]interface{})
+	if !ok {
+		t.Fatalf("expected route to contain middlewares, got %#v", route["middlewares"])
+	}
+
+	names := make([]string, 0, len(middlewares))
+	for i, middleware := range middlewares {
+		middlewareMap, ok := middleware.(map[string]interface{})
+		if !ok {
+			t.Fatalf("expected middleware %d to be an object, got %#v", i, middleware)
+		}
+
+		name, ok := middlewareMap["name"].(string)
+		if !ok {
+			t.Fatalf("expected middleware %d to contain string name, got %#v", i, middlewareMap["name"])
+		}
+
+		names = append(names, name)
+	}
+
+	return names
+}
+
 func TestIngressRouteTemplateTLSCreatesSeparateHTTPAndHTTPSRoutes(t *testing.T) {
 	docs := renderIngressRouteTemplate(t, testIngressRouteValues(true))
 	if len(docs) != 2 {
@@ -137,8 +163,8 @@ func TestIngressRouteTemplateTLSCreatesSeparateHTTPAndHTTPSRoutes(t *testing.T)
 	}
 	httpRoutes := httpSpec["routes"].([]interface{})
 	httpRoute0 := httpRoutes[0].(map[string]interface{})
-	if _, ok := httpRoute0["middlewares"]; !ok {
-		t.Fatalf("expected HTTP route to contain redirect middleware, got %#v", httpRoute0)
+	if got := middlewareNames(t, httpRoute0); strings.Join(got, ",") != "myapp-web-compression,myapp-web-redirect-to-https" {
+		t.Fatalf("expected HTTP route middlewares [myapp-web-compression myapp-web-redirect-to-https], got %#v", got)
 	}
 	if _, ok := httpSpec["tls"]; ok {
 		t.Fatalf("expected HTTP route to omit tls block, got %#v", httpSpec["tls"])
@@ -151,8 +177,8 @@ func TestIngressRouteTemplateTLSCreatesSeparateHTTPAndHTTPSRoutes(t *testing.T)
 	}
 	httpsRoutes := httpsSpec["routes"].([]interface{})
 	httpsRoute0 := httpsRoutes[0].(map[string]interface{})
-	if _, ok := httpsRoute0["middlewares"]; ok {
-		t.Fatalf("expected HTTPS route to omit redirect middleware, got %#v", httpsRoute0["middlewares"])
+	if got := middlewareNames(t, httpsRoute0); strings.Join(got, ",") != "myapp-web-compression" {
+		t.Fatalf("expected HTTPS route middlewares [myapp-web-compression], got %#v", got)
 	}
 	tls, ok := httpsSpec["tls"].(map[string]interface{})
 	if !ok {
@@ -176,8 +202,8 @@ func TestIngressRouteTemplateWithoutTLSKeepsSingleHTTPRoute(t *testing.T) {
 	}
 	httpRoutes := httpSpec["routes"].([]interface{})
 	httpRoute0 := httpRoutes[0].(map[string]interface{})
-	if _, ok := httpRoute0["middlewares"]; ok {
-		t.Fatalf("expected non-TLS route to omit redirect middleware, got %#v", httpRoute0["middlewares"])
+	if got := middlewareNames(t, httpRoute0); strings.Join(got, ",") != "myapp-web-compression" {
+		t.Fatalf("expected non-TLS route middlewares [myapp-web-compression], got %#v", got)
 	}
 	if _, ok := httpSpec["tls"]; ok {
 		t.Fatalf("expected non-TLS route to omit tls block, got %#v", httpSpec["tls"])
@@ -225,8 +251,8 @@ func TestIngressRouteTemplateMultipleDomainsRenderOneRoutePerDomain(t *testing.T
 	}
 	for i, r := range httpRoutes {
 		route := r.(map[string]interface{})
-		if _, ok := route["middlewares"]; !ok {
-			t.Fatalf("expected HTTP route entry %d to contain redirect middleware, got %#v", i, route)
+		if got := middlewareNames(t, route); strings.Join(got, ",") != "myapp-web-compression,myapp-web-redirect-to-https" {
+			t.Fatalf("expected HTTP route entry %d middlewares [myapp-web-compression myapp-web-redirect-to-https], got %#v", i, got)
 		}
 	}
 	if got := httpRoutes[0].(map[string]interface{})["match"]; got != "Host(`app.example.com`)" {
@@ -243,8 +269,8 @@ func TestIngressRouteTemplateMultipleDomainsRenderOneRoutePerDomain(t *testing.T
 	}
 	for i, r := range httpsRoutes {
 		route := r.(map[string]interface{})
-		if _, ok := route["middlewares"]; ok {
-			t.Fatalf("expected HTTPS route entry %d to omit redirect middleware, got %#v", i, route)
+		if got := middlewareNames(t, route); strings.Join(got, ",") != "myapp-web-compression" {
+			t.Fatalf("expected HTTPS route entry %d middlewares [myapp-web-compression], got %#v", i, got)
 		}
 	}
 	if got := httpsRoutes[0].(map[string]interface{})["match"]; got != "Host(`app.example.com`)" {

plugins/scheduler-k3s/templates/chart/compression-middleware.yaml

@@ -0,0 +1,27 @@
+{{- range $processName, $config := .Values.processes }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
+{{- continue }}
+{{- end }}
+
+{{- if and $config.web.domains (eq $.Values.global.network.ingress_class "traefik") }}
+---
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+    {{ include "print.annotations" (dict "config" $.Values.global "key" "traefik_middleware") | indent 4 }}
+    {{ include "print.annotations" (dict "config" $config "key" "traefik_middleware") | indent 4 }}
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}-compression
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+    {{ include "print.labels" (dict "config" $.Values.global "key" "traefik_middleware") | indent 4 }}
+    {{ include "print.labels" (dict "config" $config "key" "traefik_middleware") | indent 4 }}
+  name: {{ $.Values.global.app_name}}-{{ $processName }}-compression
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  compress: {}
+{{- end }}
+{{- end }}

plugins/scheduler-k3s/templates/chart/ingress-route.yaml

@@ -40,11 +40,13 @@ spec:
     {{- range $ddx, $domain := $config.web.domains }}
     - kind: Rule
       match: Host(`{{ $domain.name }}`)
-      {{- if and (eq $entryPoint "web") $config.web.tls.enabled }}
       middlewares:
+        - name: {{ $.Values.global.app_name}}-{{ $processName }}-compression
+          namespace: {{ $.Values.global.namespace }}
+        {{- if and (eq $entryPoint "web") $config.web.tls.enabled }}
         - name: {{ $.Values.global.app_name}}-{{ $processName }}-redirect-to-https
           namespace: {{ $.Values.global.namespace }}
-      {{- end }}
+        {{- end }}
       services:
       - name: {{ $.Values.global.app_name }}-{{ $processName }}
         namespace: {{ $.Values.global.namespace }}

plugins/scheduler-k3s/triggers.go

@@ -762,7 +762,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 
 		templateFiles := []string{"deployment", "keda-scaled-object"}
 		if processType == "web" {
-			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
+			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "compression-middleware", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
 		}
 		for _, templateName := range templateFiles {
 			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))

tests/unit/scheduler-k3s-certs-redirect.bats

@@ -63,7 +63,7 @@ teardown() {
   echo "status: $status"
   assert_success
 
-  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].middlewares[0].name}'"
+  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].middlewares[1].name}'"
   echo "output: $output"
   echo "status: $status"
   assert_success

tests/unit/scheduler-k3s-compression.bats

@@ -0,0 +1,92 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup_local_tls() {
+  TLS=$BATS_TMPDIR/tls
+  mkdir -p $TLS
+  tar xf $BATS_TEST_DIRNAME/server_ssl.tar -C $TLS
+  sudo chown -R dokku:dokku $TLS
+}
+
+teardown_local_tls() {
+  TLS=$BATS_TMPDIR/tls
+  rm -R $TLS
+}
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+  setup_local_tls
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+  teardown_local_tls
+}
+
+assert_http_localhost_header() {
+  local scheme="$1" domain="$2" port="${3:-80}" path="${4:-/}" accept_encoding="$5" expected_header="$6"
+  local retries="${HTTP_ASSERT_RETRIES:-30}" attempt=1
+
+  while [[ "$attempt" -lt "$retries" ]]; do
+    run /bin/bash -c "curl --connect-to '$domain:$port:localhost:$port' -kSsD - -o /dev/null -H 'Accept-Encoding: $accept_encoding' '$scheme://$domain:$port$path' | tr -d '\r' | grep -i '^$expected_header$'"
+    [[ "$status" -eq 0 ]] && break
+    sleep 1
+    attempt=$((attempt + 1))
+  done
+
+  echo "output: $output"
+  echo "status: $status"
+  echo "attempts: $attempt"
+  assert_success
+  assert_output_contains "$expected_header"
+}
+
+@test "(scheduler-k3s) [ingress] traefik compression middleware adds gzip response header" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=traefik install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 30"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].middlewares[0].name}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "${TEST_APP}-web-compression"
+
+  assert_http_localhost_response "https" "$TEST_APP.dokku.me" "443" "" "python/http.server"
+  assert_http_localhost_header "https" "$TEST_APP.dokku.me" "443" "/" "gzip" "content-encoding: gzip"
+}