Merge pull request #6547 from dokku/proxy-implementation

docs/deployment/schedulers/k3s.md

@@ -55,6 +55,14 @@ By default, Dokku will attempt to auto-detect the IP address of the server. In c
 dokku scheduler-k3s:initialize --server-ip 192.168.20.15
 ```
 
+Dokku's k3s integration natively uses `Traefik` as it's ingress load balancer via [Traefik's CRDs](https://doc.traefik.io/traefik/providers/kubernetes-crd/), but a cluster can be set to use `nginx` via the [ingress-nginx](https://github.com/kubernetes/ingress-nginx) project. Switching to nginx will result in any `nginx` plugin settings being respected, either by turning them into annotations or creating a custom server snippet.
+
+To change the ingress, set the `--ingress-class` flag:
+
+```shell
+dokku scheduler-k3s:initialize --ingress-class nginx
+```
+
 ### Adding nodes to the cluster
 
 > [!WARNING]

plugins/scheduler-k3s/functions.go

@@ -14,6 +14,7 @@ import (
 
 	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
+	nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts"
 	resty "github.com/go-resty/resty/v2"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/yaml.v3"
@@ -384,7 +385,7 @@ func getAnnotations(appName string, processType string) (ProcessAnnotations, err
 	}
 	annotations.DeploymentAnnotations = deploymentAnnotations
 
-	ingressAnnotations, err := getAnnotation(appName, processType, "ingress")
+	ingressAnnotations, err := getIngressAnnotations(appName, processType)
 	if err != nil {
 		return annotations, err
 	}
@@ -494,6 +495,179 @@ func getComputedImagePullSecrets(appName string) string {
 	return imagePullSecrets
 }
 
+func getGlobalIngressClass() string {
+	return common.PropertyGetDefault("scheduler-k3s", "global", "ingress-class", DefaultIngressClass)
+}
+
+func getIngressAnnotations(appName string, processType string) (map[string]string, error) {
+	type annotation struct {
+		annotation      string
+		getter          func(appName string) string
+		locationSnippet func(value string) string
+		serverSnippet   func(value string) string
+	}
+
+	locationLines := []string{}
+	serverLines := []string{}
+
+	properties := map[string]annotation{
+		"access-log-path": {
+			getter: nginxvhosts.ComputedAccessLogPath,
+			serverSnippet: func(value string) string {
+				if value == "" {
+					return ""
+				}
+				return fmt.Sprintf("access_log %s;", value)
+			},
+		},
+		"bind-address-ipv4": {
+			getter: nginxvhosts.ComputedBindAddressIPv4,
+		},
+		"bind-address-ipv6": {
+			getter: nginxvhosts.ComputedBindAddressIPv6,
+		},
+		"client-max-body-size": {
+			annotation: "nginx.ingress.kubernetes.io/proxy-body-size",
+			getter:     nginxvhosts.ComputedClientMaxBodySize,
+		},
+		"disable-custom-config": {
+			getter: nginxvhosts.ComputedDisableCustomConfig,
+		},
+		"error-log-path": {
+			getter: nginxvhosts.ComputedErrorLogPath,
+			serverSnippet: func(value string) string {
+				if value == "" {
+					return ""
+				}
+				return fmt.Sprintf("error_log %s;", value)
+			},
+		},
+		// todo: handle hsts properly
+		"hsts-include-subdomains": {
+			getter: nginxvhosts.ComputedHSTSIncludeSubdomains,
+		},
+		"hsts-max-age": {
+			getter: nginxvhosts.ComputedHSTSMaxAge,
+		},
+		"hsts-preload": {
+			getter: nginxvhosts.ComputedHSTSPreload,
+		},
+		"hsts": {
+			getter: nginxvhosts.ComputedHSTS,
+		},
+		"nginx-conf-sigil-path": {
+			getter: nginxvhosts.ComputedNginxConfSigilPath,
+		},
+		"proxy-buffer-size": {
+			annotation: "nginx.ingress.kubernetes.io/proxy-buffer-size",
+			getter:     nginxvhosts.ComputedProxyBufferSize,
+		},
+		"proxy-buffering": {
+			annotation: "nginx.ingress.kubernetes.io/proxy-buffering",
+			getter:     nginxvhosts.ComputedProxyBuffering,
+		},
+		"proxy-buffers": {
+			annotation: "nginx.ingress.kubernetes.io/proxy-buffers-number",
+			getter:     nginxvhosts.ComputedProxyBuffers,
+		},
+		"proxy-busy-buffers-size": {
+			getter: nginxvhosts.ComputedProxyBusyBuffersSize,
+			locationSnippet: func(value string) string {
+				if value == "" {
+					return ""
+				}
+				return fmt.Sprintf("proxy_busy_buffers_size %s;", value)
+			},
+		},
+		"proxy-read-timeout": {
+			annotation: "nginx.ingress.kubernetes.io/proxy-read-timeout",
+			getter:     nginxvhosts.ComputedProxyReadTimeout,
+		},
+		"x-forwarded-for-value": {
+			getter: nginxvhosts.ComputedXForwardedForValue,
+			locationSnippet: func(value string) string {
+				if value == "" {
+					return ""
+				}
+				return fmt.Sprintf("proxy_set_header X-Forwarded-For %s;", value)
+			},
+		},
+		"x-forwarded-port-value": {
+			getter: nginxvhosts.ComputedXForwardedPortValue,
+			locationSnippet: func(value string) string {
+				if value == "" {
+					return ""
+				}
+				return fmt.Sprintf("proxy_set_header X-Forwarded-Port %s;", value)
+			},
+		},
+		"x-forwarded-proto-value": {
+			getter: nginxvhosts.ComputedXForwardedProtoValue,
+			locationSnippet: func(value string) string {
+				if value == "" {
+					return ""
+				}
+				return fmt.Sprintf("proxy_set_header X-Forwarded-Proto %s;", value)
+			},
+		},
+		"x-forwarded-ssl": {
+			getter: nginxvhosts.ComputedXForwardedSSL,
+			locationSnippet: func(value string) string {
+				if value == "" {
+					return ""
+				}
+				return fmt.Sprintf("proxy_set_header X-Forwarded-SSL %s;", value)
+			},
+		},
+	}
+
+	annotations := map[string]string{}
+	for _, newKey := range properties {
+		if newKey.locationSnippet != nil {
+			locationLines = append(locationLines, newKey.locationSnippet(newKey.getter(appName)))
+		} else if newKey.serverSnippet != nil {
+			serverLines = append(serverLines, newKey.serverSnippet(newKey.getter(appName)))
+		} else if newKey.annotation != "" {
+			annotations[newKey.annotation] = newKey.getter(appName)
+		}
+	}
+
+	var locationSnippet string
+	for _, line := range locationLines {
+		if line != "" {
+			locationSnippet += line + "\n"
+		}
+	}
+	var serverSnippet string
+	for _, line := range serverLines {
+		if line != "" {
+			serverSnippet += line + "\n"
+		}
+	}
+
+	if locationSnippet != "" {
+		annotations["nginx.ingress.kubernetes.io/configuration-snippet"] = locationSnippet
+	}
+	if serverSnippet != "" {
+		annotations["nginx.ingress.kubernetes.io/server-snippet"] = serverSnippet
+	}
+
+	customAnnotations, err := getAnnotation(appName, processType, "deployment")
+	if err != nil {
+		return map[string]string{}, err
+	}
+
+	for key, value := range customAnnotations {
+		if _, ok := annotations[key]; ok {
+			common.LogWarn(fmt.Sprintf("Nginx-based annotation %s will be overwritten by custom annotation", key))
+		}
+
+		annotations[key] = value
+	}
+
+	return annotations, nil
+}
+
 func getLetsencryptServer(appName string) string {
 	return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "")
 }
@@ -754,7 +928,7 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
-func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
+func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error {
 	for _, repo := range HelmRepositories {
 		helmAgent, err := NewHelmAgent("default", DeployLogPrinter)
 		if err != nil {
@@ -768,6 +942,10 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
 	}
 
 	for _, chart := range HelmCharts {
+		if !shouldInstall(chart) {
+			continue
+		}
+
 		if chart.CreateNamespace {
 			namespace := corev1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{
@@ -873,6 +1051,65 @@ func installHelperCommands(ctx context.Context) error {
 		}
 	}
 
+	return installHelm(ctx)
+}
+
+func installHelm(ctx context.Context) error {
+	client := resty.New()
+	resp, err := client.R().
+		SetContext(ctx).
+		Get("https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3")
+	if err != nil {
+		return fmt.Errorf("Unable to download helm installer: %w", err)
+	}
+	if resp == nil {
+		return fmt.Errorf("Missing response from helm installer download: %w", err)
+	}
+
+	if resp.StatusCode() != 200 {
+		return fmt.Errorf("Invalid status code for helm installer script: %d", resp.StatusCode())
+	}
+
+	f, err := os.CreateTemp("", "sample")
+	if err != nil {
+		return fmt.Errorf("Unable to create temporary file for helm installer: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("Unable to close helm installer file: %w", err)
+	}
+
+	err = common.WriteStringToFile(common.WriteStringToFileInput{
+		Content:  resp.String(),
+		Filename: f.Name(),
+		Mode:     os.FileMode(0755),
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to write helm installer to file: %w", err)
+	}
+
+	fi, err := os.Stat(f.Name())
+	if err != nil {
+		return fmt.Errorf("Unable to get helm installer file size: %w", err)
+	}
+
+	if fi.Size() == 0 {
+		return fmt.Errorf("Invalid helm installer filesize")
+	}
+
+	common.LogInfo2Quiet("Running helm installer")
+	installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command:     f.Name(),
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call helm installer command: %w", err)
+	}
+	if installerCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from helm installer command: %d", installerCmd.ExitCode)
+	}
+
 	return nil
 }
 

plugins/scheduler-k3s/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
+	github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-00010101000000-000000000000
 	github.com/fatih/color v1.16.0
 	github.com/go-openapi/jsonpointer v0.19.6
 	github.com/go-resty/resty/v2 v2.11.0
@@ -18,6 +19,7 @@ require (
 	github.com/ryanuber/columnize v2.1.2+incompatible
 	github.com/spf13/pflag v1.0.5
 	github.com/traefik/traefik/v2 v2.10.7
+	golang.org/x/sync v0.6.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.13.3
 	k8s.io/api v0.29.0
@@ -152,7 +154,6 @@ require (
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.12.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
@@ -186,3 +187,5 @@ replace github.com/dokku/dokku/plugins/common => ../common
 replace github.com/dokku/dokku/plugins/config => ../config
 
 replace github.com/dokku/dokku/plugins/cron => ../cron
+
+replace github.com/dokku/dokku/plugins/nginx-vhosts => ../nginx-vhosts

plugins/scheduler-k3s/report.go

@@ -20,6 +20,7 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		"--scheduler-k3s-computed-letsencrypt-server":   reportComputedLetsencryptServer,
 		"--scheduler-k3s-letsencrypt-server":            reportLetsencryptServer,
 		"--scheduler-k3s-global-letsencrypt-server":     reportGlobalLetsencryptServer,
+		"--scheduler-k3s-global-ingress-class":          reportGlobalIngressClass,
 		"--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd,
 		"--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag,
 		"--scheduler-k3s-computed-namespace":            reportComputedNamespace,
@@ -66,6 +67,10 @@ func reportGlobalImagePullSecrets(appName string) string {
 	return getGlobalImagePullSecrets()
 }
 
+func reportGlobalIngressClass(appName string) string {
+	return getGlobalIngressClass()
+}
+
 func reportComputedLetsencryptServer(appName string) string {
 	return getComputedLetsencryptServer(appName)
 }

plugins/scheduler-k3s/scheduler_k3s.go

@@ -28,6 +28,7 @@ var (
 	GlobalProperties = map[string]bool{
 		"deploy-timeout":         true,
 		"image-pull-secrets":     true,
+		"ingress-class":          true,
 		"letsencrypt-server":     true,
 		"letsencrypt-email-prod": true,
 		"letsencrypt-email-stag": true,
@@ -38,11 +39,10 @@ var (
 	}
 )
 
-const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
-
-const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml"
-
+const DefaultIngressClass = "traefik"
 const GlobalProcessType = "--global"
+const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
+const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml"
 
 var (
 	runtimeScheme  = runtime.NewScheme()
@@ -68,13 +68,13 @@ var KubernetesManifests = []Manifest{
 }
 
 type HelmChart struct {
-	ReleaseName     string
 	ChartPath       string
-	Namespace       string
 	CreateNamespace bool
-	Version         string
+	Namespace       string
 	Path            string
+	ReleaseName     string
 	RepoURL         string
+	Version         string
 }
 
 var HelmCharts = []HelmChart{
@@ -102,6 +102,14 @@ var HelmCharts = []HelmChart{
 		RepoURL:         "https://helm.traefik.io/traefik",
 		Version:         "26.0.0",
 	},
+	{
+		ChartPath:       "ingress-nginx",
+		CreateNamespace: true,
+		Namespace:       "ingress-nginx",
+		ReleaseName:     "ingress-nginx",
+		RepoURL:         "https://kubernetes.github.io/ingress-nginx",
+		Version:         "4.7.5",
+	},
 }
 
 type HelmRepository struct {

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -22,8 +22,9 @@ func main() {
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
 		serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
+		ingressClass := args.String("ingress-class", "traefik", "ingress-class: ingress-class to use for all outbound traffic")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandInitialize(*serverIP, *taintScheduling)
+		err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling)
 	case "cluster-add":
 		args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError)
 		allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts")

plugins/scheduler-k3s/subcommands.go

@@ -59,7 +59,11 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri
 }
 
 // CommandInitialize initializes a k3s cluster on the local server
-func CommandInitialize(serverIP string, taintScheduling bool) error {
+func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error {
+	if ingressClass != "nginx" && ingressClass != "traefik" {
+		return fmt.Errorf("Invalid ingress-class: %s", ingressClass)
+	}
+
 	if err := isK3sInstalled(); err == nil {
 		return fmt.Errorf("k3s already installed, cannot re-initialize k3s")
 	}
@@ -219,6 +223,11 @@ func CommandInitialize(serverIP string, taintScheduling bool) error {
 		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
 	}
 
+	common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties)
+	if ingressClass == "nginx" {
+		args = append(args, "--disable", "traefik")
+	}
+
 	common.LogInfo2Quiet("Running k3s installer")
 	installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
 		Command:     f.Name(),
@@ -298,7 +307,17 @@ func CommandInitialize(serverIP string, taintScheduling bool) error {
 	}
 
 	common.LogInfo2Quiet("Installing helm charts")
-	err = installHelmCharts(ctx, clientset)
+	err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool {
+		if chart.ChartPath == "traefik" && ingressClass == "nginx" {
+			return false
+		}
+
+		if chart.ChartPath == "nginx" && ingressClass == "traefik" {
+			return false
+		}
+
+		return true
+	})
 	if err != nil {
 		return fmt.Errorf("Unable to install helm charts: %w", err)
 	}

plugins/scheduler-k3s/template.go

@@ -73,8 +73,8 @@ type ProcessAnnotations struct {
 	JobAnnotations                 map[string]string `yaml:"job,omitempty"`
 	PodAnnotations                 map[string]string `yaml:"pod,omitempty"`
 	SecretAnnotations              map[string]string `yaml:"secret,omitempty"`
-	ServiceAnnotations             map[string]string `yaml:"service,omitempty"`
 	ServiceAccountAnnotations      map[string]string `yaml:"serviceaccount,omitempty"`
+	ServiceAnnotations             map[string]string `yaml:"service,omitempty"`
 	TraefikIngressRouteAnnotations map[string]string `yaml:"traefik_ingressroute,omitempty"`
 	TraefikMiddlewareAnnotations   map[string]string `yaml:"traefik_middleware,omitempty"`
 }

plugins/scheduler-k3s/templates/chart/ingress.yaml

@@ -0,0 +1,51 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := index .Values.processes "PROCESS_NAME" }}
+{{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }}
+{{- range $pdx, $port_map := $config.web.port_maps }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+    {{- if and $.Values.global.annotations $.Values.global.annotations.deployment }}
+    {{- range $k, $v := $.Values.global.annotations.deployment }}
+    {{ $k }}: {{ $v | quote }}
+    {{- end }}
+    {{- end }}
+    {{- if and $config.annotations $config.annotations.ingress }}
+    {{- range $k, $v := $config.annotations.ingress }}
+    {{ $k }}: {{ $v | quote }}
+    {{- end }}
+    {{- end }}
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }}
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  ingressClassName: nginx
+  {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }}
+  tls:
+    - hosts:
+    {{- range $ddx, $domain := $config.web.domains }}
+      - {{ $domain | quote }}
+    {{- end }}
+    secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
+  {{- end }}
+  rules:
+    {{- range $ddx, $domain := $config.web.domains }}
+    - host: {{ $domain | quote }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: {{ $.Values.global.app_name }}-{{ $processName }}
+                port:
+                  name: {{ $port_map.name }}
+            pathType: ImplementationSpecific
+            path: /
+    {{- end }}
+{{- end }}
+{{- end }}

plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml

@@ -0,0 +1,57 @@
+---
+controller:
+  metrics:
+    enabled: true
+  config:
+    access-log-path: /var/log/nginx/access.log
+    error-log-path: /var/log/nginx/error.log
+    log-format-escape-json: "true"
+  extraVolumeMounts:
+    - name: data
+      mountPath: /var/log/nginx
+  extraVolumes:
+    - name: data
+      emptyDir: {}
+  extraContainers:
+    - name: stream-accesslog
+      image: busybox
+      args:
+        - /bin/sh
+        - -c
+        - tail -n+1 -F /var/log/nginx/access.log
+      imagePullPolicy: Always
+      resources: {}
+      terminationMessagePath: /dev/termination-log
+      terminationMessagePolicy: File
+      volumeMounts:
+        - mountPath: /var/log/nginx
+          name: data
+    - name: stream-errorlog
+      image: busybox
+      args:
+        - /bin/sh
+        - -c
+        - tail -n+1 -F /var/log/nginx/error.log
+      imagePullPolicy: Always
+      resources: {}
+      terminationMessagePath: /dev/termination-log
+      terminationMessagePolicy: File
+      volumeMounts:
+        - mountPath: /var/log/nginx
+          name: data
+  allowSnippetAnnotations: true
+  podAnnotations:
+    prometheus.io/path: /metrics
+    prometheus.io/port: "10254"
+    prometheus.io/scheme: http
+    prometheus.io/scrape: "true"
+  tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+    - key: node-role.kubernetes.io/master
+      operator: Exists
+      effect: NoSchedule
+  priorityClassName: system-cluster-critical

plugins/scheduler-k3s/triggers.go

@@ -351,7 +351,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			},
 			Namespace: namespace,
 			Network: GlobalNetwork{
-				IngressClass: "traefik",
+				IngressClass: common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass),
 				PrimaryPort:  primaryPort,
 			},
 			Secrets: map[string]string{},
@@ -439,7 +439,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 
 		templateFiles := []string{"deployment"}
 		if processType == "web" {
-			templateFiles = append(templateFiles, "service", "certificate", "ingress-route", "https-redirect-middleware")
+			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware")
 		}
 		for _, templateName := range templateFiles {
 			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))

tests/unit/scheduler-k3s.bats

@@ -49,7 +49,8 @@ install_k3s() {
   assert_success
   assert_output_contains "Login Succeeded"
 
-  run /bin/bash -c "dokku scheduler-k3s:initialize"
+  INGRESS_CLASS="${INGRESS_CLASS:-traefik}"
+  run /bin/bash -c "dokku scheduler-k3s:initialize --ingress-class $INGRESS_CLASS"
   echo "output: $output"
   echo "status: $status"
   assert_success
@@ -70,12 +71,42 @@ uninstall_k3s() {
   install_k3s
 }
 
-@test "(scheduler-k3s) deploy" {
+@test "(scheduler-k3s) deploy traefik" {
   if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
     skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
   fi
 
-  install_k3s
+  INGRESS_CLASS=traefik install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 20"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
+}
+
+@test "(scheduler-k3s) deploy nginx" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
 
   run /bin/bash -c "dokku apps:create $TEST_APP"
   echo "output: $output"