diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 6054812b8..540bd157d 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -55,6 +55,14 @@ By default, Dokku will attempt to auto-detect the IP address of the server. In c dokku scheduler-k3s:initialize --server-ip 192.168.20.15 ``` +Dokku's k3s integration natively uses `Traefik` as it's ingress load balancer via [Traefik's CRDs](https://doc.traefik.io/traefik/providers/kubernetes-crd/), but a cluster can be set to use `nginx` via the [ingress-nginx](https://github.com/kubernetes/ingress-nginx) project. Switching to nginx will result in any `nginx` plugin settings being respected, either by turning them into annotations or creating a custom server snippet. + +To change the ingress, set the `--ingress-class` flag: + +```shell +dokku scheduler-k3s:initialize --ingress-class nginx +``` + ### Adding nodes to the cluster > [!WARNING] diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index c783d4502..5571cb19e 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -14,6 +14,7 @@ import ( appjson "github.com/dokku/dokku/plugins/app-json" "github.com/dokku/dokku/plugins/common" + nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts" resty "github.com/go-resty/resty/v2" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" @@ -384,7 +385,7 @@ func getAnnotations(appName string, processType string) (ProcessAnnotations, err } annotations.DeploymentAnnotations = deploymentAnnotations - ingressAnnotations, err := getAnnotation(appName, processType, "ingress") + ingressAnnotations, err := getIngressAnnotations(appName, processType) if err != nil { return annotations, err } @@ -494,6 +495,179 @@ func getComputedImagePullSecrets(appName string) string { return imagePullSecrets } +func getGlobalIngressClass() string { + return common.PropertyGetDefault("scheduler-k3s", "global", "ingress-class", DefaultIngressClass) +} + +func getIngressAnnotations(appName string, processType string) (map[string]string, error) { + type annotation struct { + annotation string + getter func(appName string) string + locationSnippet func(value string) string + serverSnippet func(value string) string + } + + locationLines := []string{} + serverLines := []string{} + + properties := map[string]annotation{ + "access-log-path": { + getter: nginxvhosts.ComputedAccessLogPath, + serverSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("access_log %s;", value) + }, + }, + "bind-address-ipv4": { + getter: nginxvhosts.ComputedBindAddressIPv4, + }, + "bind-address-ipv6": { + getter: nginxvhosts.ComputedBindAddressIPv6, + }, + "client-max-body-size": { + annotation: "nginx.ingress.kubernetes.io/proxy-body-size", + getter: nginxvhosts.ComputedClientMaxBodySize, + }, + "disable-custom-config": { + getter: nginxvhosts.ComputedDisableCustomConfig, + }, + "error-log-path": { + getter: nginxvhosts.ComputedErrorLogPath, + serverSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("error_log %s;", value) + }, + }, + // todo: handle hsts properly + "hsts-include-subdomains": { + getter: nginxvhosts.ComputedHSTSIncludeSubdomains, + }, + "hsts-max-age": { + getter: nginxvhosts.ComputedHSTSMaxAge, + }, + "hsts-preload": { + getter: nginxvhosts.ComputedHSTSPreload, + }, + "hsts": { + getter: nginxvhosts.ComputedHSTS, + }, + "nginx-conf-sigil-path": { + getter: nginxvhosts.ComputedNginxConfSigilPath, + }, + "proxy-buffer-size": { + annotation: "nginx.ingress.kubernetes.io/proxy-buffer-size", + getter: nginxvhosts.ComputedProxyBufferSize, + }, + "proxy-buffering": { + annotation: "nginx.ingress.kubernetes.io/proxy-buffering", + getter: nginxvhosts.ComputedProxyBuffering, + }, + "proxy-buffers": { + annotation: "nginx.ingress.kubernetes.io/proxy-buffers-number", + getter: nginxvhosts.ComputedProxyBuffers, + }, + "proxy-busy-buffers-size": { + getter: nginxvhosts.ComputedProxyBusyBuffersSize, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_busy_buffers_size %s;", value) + }, + }, + "proxy-read-timeout": { + annotation: "nginx.ingress.kubernetes.io/proxy-read-timeout", + getter: nginxvhosts.ComputedProxyReadTimeout, + }, + "x-forwarded-for-value": { + getter: nginxvhosts.ComputedXForwardedForValue, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-For %s;", value) + }, + }, + "x-forwarded-port-value": { + getter: nginxvhosts.ComputedXForwardedPortValue, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-Port %s;", value) + }, + }, + "x-forwarded-proto-value": { + getter: nginxvhosts.ComputedXForwardedProtoValue, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-Proto %s;", value) + }, + }, + "x-forwarded-ssl": { + getter: nginxvhosts.ComputedXForwardedSSL, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-SSL %s;", value) + }, + }, + } + + annotations := map[string]string{} + for _, newKey := range properties { + if newKey.locationSnippet != nil { + locationLines = append(locationLines, newKey.locationSnippet(newKey.getter(appName))) + } else if newKey.serverSnippet != nil { + serverLines = append(serverLines, newKey.serverSnippet(newKey.getter(appName))) + } else if newKey.annotation != "" { + annotations[newKey.annotation] = newKey.getter(appName) + } + } + + var locationSnippet string + for _, line := range locationLines { + if line != "" { + locationSnippet += line + "\n" + } + } + var serverSnippet string + for _, line := range serverLines { + if line != "" { + serverSnippet += line + "\n" + } + } + + if locationSnippet != "" { + annotations["nginx.ingress.kubernetes.io/configuration-snippet"] = locationSnippet + } + if serverSnippet != "" { + annotations["nginx.ingress.kubernetes.io/server-snippet"] = serverSnippet + } + + customAnnotations, err := getAnnotation(appName, processType, "deployment") + if err != nil { + return map[string]string{}, err + } + + for key, value := range customAnnotations { + if _, ok := annotations[key]; ok { + common.LogWarn(fmt.Sprintf("Nginx-based annotation %s will be overwritten by custom annotation", key)) + } + + annotations[key] = value + } + + return annotations, nil +} + func getLetsencryptServer(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "") } @@ -754,7 +928,7 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { }, nil } -func installHelmCharts(ctx context.Context, clientset KubernetesClient) error { +func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error { for _, repo := range HelmRepositories { helmAgent, err := NewHelmAgent("default", DeployLogPrinter) if err != nil { @@ -768,6 +942,10 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error { } for _, chart := range HelmCharts { + if !shouldInstall(chart) { + continue + } + if chart.CreateNamespace { namespace := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -873,6 +1051,65 @@ func installHelperCommands(ctx context.Context) error { } } + return installHelm(ctx) +} + +func installHelm(ctx context.Context) error { + client := resty.New() + resp, err := client.R(). + SetContext(ctx). + Get("https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3") + if err != nil { + return fmt.Errorf("Unable to download helm installer: %w", err) + } + if resp == nil { + return fmt.Errorf("Missing response from helm installer download: %w", err) + } + + if resp.StatusCode() != 200 { + return fmt.Errorf("Invalid status code for helm installer script: %d", resp.StatusCode()) + } + + f, err := os.CreateTemp("", "sample") + if err != nil { + return fmt.Errorf("Unable to create temporary file for helm installer: %w", err) + } + defer os.Remove(f.Name()) + + if err := f.Close(); err != nil { + return fmt.Errorf("Unable to close helm installer file: %w", err) + } + + err = common.WriteStringToFile(common.WriteStringToFileInput{ + Content: resp.String(), + Filename: f.Name(), + Mode: os.FileMode(0755), + }) + if err != nil { + return fmt.Errorf("Unable to write helm installer to file: %w", err) + } + + fi, err := os.Stat(f.Name()) + if err != nil { + return fmt.Errorf("Unable to get helm installer file size: %w", err) + } + + if fi.Size() == 0 { + return fmt.Errorf("Invalid helm installer filesize") + } + + common.LogInfo2Quiet("Running helm installer") + installerCmd, err := common.CallExecCommand(common.ExecCommandInput{ + Command: f.Name(), + StreamStdio: true, + }) + if err != nil { + return fmt.Errorf("Unable to call helm installer command: %w", err) + } + if installerCmd.ExitCode != 0 { + return fmt.Errorf("Invalid exit code from helm installer command: %d", installerCmd.ExitCode) + } + return nil } diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index ba0ea0ab4..9ef162d0d 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -8,6 +8,7 @@ require ( github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000 + github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-00010101000000-000000000000 github.com/fatih/color v1.16.0 github.com/go-openapi/jsonpointer v0.19.6 github.com/go-resty/resty/v2 v2.11.0 @@ -18,6 +19,7 @@ require ( github.com/ryanuber/columnize v2.1.2+incompatible github.com/spf13/pflag v1.0.5 github.com/traefik/traefik/v2 v2.10.7 + golang.org/x/sync v0.6.0 gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.13.3 k8s.io/api v0.29.0 @@ -152,7 +154,6 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -186,3 +187,5 @@ replace github.com/dokku/dokku/plugins/common => ../common replace github.com/dokku/dokku/plugins/config => ../config replace github.com/dokku/dokku/plugins/cron => ../cron + +replace github.com/dokku/dokku/plugins/nginx-vhosts => ../nginx-vhosts diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index 1974114fa..25474883b 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -20,6 +20,7 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-computed-letsencrypt-server": reportComputedLetsencryptServer, "--scheduler-k3s-letsencrypt-server": reportLetsencryptServer, "--scheduler-k3s-global-letsencrypt-server": reportGlobalLetsencryptServer, + "--scheduler-k3s-global-ingress-class": reportGlobalIngressClass, "--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd, "--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag, "--scheduler-k3s-computed-namespace": reportComputedNamespace, @@ -66,6 +67,10 @@ func reportGlobalImagePullSecrets(appName string) string { return getGlobalImagePullSecrets() } +func reportGlobalIngressClass(appName string) string { + return getGlobalIngressClass() +} + func reportComputedLetsencryptServer(appName string) string { return getComputedLetsencryptServer(appName) } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 8b57e389a..78a857fa4 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -28,6 +28,7 @@ var ( GlobalProperties = map[string]bool{ "deploy-timeout": true, "image-pull-secrets": true, + "ingress-class": true, "letsencrypt-server": true, "letsencrypt-email-prod": true, "letsencrypt-email-stag": true, @@ -38,11 +39,10 @@ var ( } ) -const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" - -const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml" - +const DefaultIngressClass = "traefik" const GlobalProcessType = "--global" +const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" +const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml" var ( runtimeScheme = runtime.NewScheme() @@ -68,13 +68,13 @@ var KubernetesManifests = []Manifest{ } type HelmChart struct { - ReleaseName string ChartPath string - Namespace string CreateNamespace bool - Version string + Namespace string Path string + ReleaseName string RepoURL string + Version string } var HelmCharts = []HelmChart{ @@ -102,6 +102,14 @@ var HelmCharts = []HelmChart{ RepoURL: "https://helm.traefik.io/traefik", Version: "26.0.0", }, + { + ChartPath: "ingress-nginx", + CreateNamespace: true, + Namespace: "ingress-nginx", + ReleaseName: "ingress-nginx", + RepoURL: "https://kubernetes.github.io/ingress-nginx", + Version: "4.7.5", + }, } type HelmRepository struct { diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 38616f7d5..cf45bf252 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -22,8 +22,9 @@ func main() { args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node") + ingressClass := args.String("ingress-class", "traefik", "ingress-class: ingress-class to use for all outbound traffic") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandInitialize(*serverIP, *taintScheduling) + err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling) case "cluster-add": args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError) allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index ac7867eaa..5a72476d9 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -59,7 +59,11 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri } // CommandInitialize initializes a k3s cluster on the local server -func CommandInitialize(serverIP string, taintScheduling bool) error { +func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error { + if ingressClass != "nginx" && ingressClass != "traefik" { + return fmt.Errorf("Invalid ingress-class: %s", ingressClass) + } + if err := isK3sInstalled(); err == nil { return fmt.Errorf("k3s already installed, cannot re-initialize k3s") } @@ -219,6 +223,11 @@ func CommandInitialize(serverIP string, taintScheduling bool) error { args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule") } + common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties) + if ingressClass == "nginx" { + args = append(args, "--disable", "traefik") + } + common.LogInfo2Quiet("Running k3s installer") installerCmd, err := common.CallExecCommand(common.ExecCommandInput{ Command: f.Name(), @@ -298,7 +307,17 @@ func CommandInitialize(serverIP string, taintScheduling bool) error { } common.LogInfo2Quiet("Installing helm charts") - err = installHelmCharts(ctx, clientset) + err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool { + if chart.ChartPath == "traefik" && ingressClass == "nginx" { + return false + } + + if chart.ChartPath == "nginx" && ingressClass == "traefik" { + return false + } + + return true + }) if err != nil { return fmt.Errorf("Unable to install helm charts: %w", err) } diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index b9c76c96a..c371daae8 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -73,8 +73,8 @@ type ProcessAnnotations struct { JobAnnotations map[string]string `yaml:"job,omitempty"` PodAnnotations map[string]string `yaml:"pod,omitempty"` SecretAnnotations map[string]string `yaml:"secret,omitempty"` - ServiceAnnotations map[string]string `yaml:"service,omitempty"` ServiceAccountAnnotations map[string]string `yaml:"serviceaccount,omitempty"` + ServiceAnnotations map[string]string `yaml:"service,omitempty"` TraefikIngressRouteAnnotations map[string]string `yaml:"traefik_ingressroute,omitempty"` TraefikMiddlewareAnnotations map[string]string `yaml:"traefik_middleware,omitempty"` } diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml new file mode 100644 index 000000000..25c9cd068 --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -0,0 +1,51 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := index .Values.processes "PROCESS_NAME" }} +{{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} +{{- range $pdx, $port_map := $config.web.port_maps }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.deployment }} + {{- range $k, $v := $.Values.global.annotations.deployment }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.ingress }} + {{- range $k, $v := $config.annotations.ingress }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }} + namespace: {{ $.Values.global.namespace }} +spec: + ingressClassName: nginx + {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }} + tls: + - hosts: + {{- range $ddx, $domain := $config.web.domains }} + - {{ $domain | quote }} + {{- end }} + secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }} + {{- end }} + rules: + {{- range $ddx, $domain := $config.web.domains }} + - host: {{ $domain | quote }} + http: + paths: + - backend: + service: + name: {{ $.Values.global.app_name }}-{{ $processName }} + port: + name: {{ $port_map.name }} + pathType: ImplementationSpecific + path: / + {{- end }} +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml new file mode 100644 index 000000000..8263f6d42 --- /dev/null +++ b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml @@ -0,0 +1,57 @@ +--- +controller: + metrics: + enabled: true + config: + access-log-path: /var/log/nginx/access.log + error-log-path: /var/log/nginx/error.log + log-format-escape-json: "true" + extraVolumeMounts: + - name: data + mountPath: /var/log/nginx + extraVolumes: + - name: data + emptyDir: {} + extraContainers: + - name: stream-accesslog + image: busybox + args: + - /bin/sh + - -c + - tail -n+1 -F /var/log/nginx/access.log + imagePullPolicy: Always + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log/nginx + name: data + - name: stream-errorlog + image: busybox + args: + - /bin/sh + - -c + - tail -n+1 -F /var/log/nginx/error.log + imagePullPolicy: Always + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log/nginx + name: data + allowSnippetAnnotations: true + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scheme: http + prometheus.io/scrape: "true" + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + priorityClassName: system-cluster-critical diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 723cf81b7..16f824271 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -351,7 +351,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e }, Namespace: namespace, Network: GlobalNetwork{ - IngressClass: "traefik", + IngressClass: common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass), PrimaryPort: primaryPort, }, Secrets: map[string]string{}, @@ -439,7 +439,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e templateFiles := []string{"deployment"} if processType == "web" { - templateFiles = append(templateFiles, "service", "certificate", "ingress-route", "https-redirect-middleware") + templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware") } for _, templateName := range templateFiles { b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) diff --git a/tests/unit/scheduler-k3s.bats b/tests/unit/scheduler-k3s.bats index ca503a258..182f489e7 100644 --- a/tests/unit/scheduler-k3s.bats +++ b/tests/unit/scheduler-k3s.bats @@ -49,7 +49,8 @@ install_k3s() { assert_success assert_output_contains "Login Succeeded" - run /bin/bash -c "dokku scheduler-k3s:initialize" + INGRESS_CLASS="${INGRESS_CLASS:-traefik}" + run /bin/bash -c "dokku scheduler-k3s:initialize --ingress-class $INGRESS_CLASS" echo "output: $output" echo "status: $status" assert_success @@ -70,12 +71,42 @@ uninstall_k3s() { install_k3s } -@test "(scheduler-k3s) deploy" { +@test "(scheduler-k3s) deploy traefik" { if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" fi - install_k3s + INGRESS_CLASS=traefik install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 20" + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + +@test "(scheduler-k3s) deploy nginx" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s run /bin/bash -c "dokku apps:create $TEST_APP" echo "output: $output"