From 81286a067362ed6232d65510d9b700dc083da998 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 30 Jan 2024 10:20:12 -0500 Subject: [PATCH 01/12] feat: add ability to install nginx instead of traefik --- plugins/scheduler-k3s/functions.go | 65 ++++++++++++++++++- plugins/scheduler-k3s/scheduler_k3s.go | 15 ++++- .../src/subcommands/subcommands.go | 3 +- plugins/scheduler-k3s/subcommands.go | 23 ++++++- .../templates/helm-config/ingress-nginx.yaml | 45 +++++++++++++ 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index c783d4502..f2111ced7 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -754,7 +754,7 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { }, nil } -func installHelmCharts(ctx context.Context, clientset KubernetesClient) error { +func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error { for _, repo := range HelmRepositories { helmAgent, err := NewHelmAgent("default", DeployLogPrinter) if err != nil { @@ -768,6 +768,10 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error { } for _, chart := range HelmCharts { + if !shouldInstall(chart) { + continue + } + if chart.CreateNamespace { namespace := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -873,6 +877,65 @@ func installHelperCommands(ctx context.Context) error { } } + return installHelm(ctx) +} + +func installHelm(ctx context.Context) error { + client := resty.New() + resp, err := client.R(). + SetContext(ctx). + Get("https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3") + if err != nil { + return fmt.Errorf("Unable to download helm installer: %w", err) + } + if resp == nil { + return fmt.Errorf("Missing response from helm installer download: %w", err) + } + + if resp.StatusCode() != 200 { + return fmt.Errorf("Invalid status code for helm installer script: %d", resp.StatusCode()) + } + + f, err := os.CreateTemp("", "sample") + if err != nil { + return fmt.Errorf("Unable to create temporary file for helm installer: %w", err) + } + defer os.Remove(f.Name()) + + if err := f.Close(); err != nil { + return fmt.Errorf("Unable to close helm installer file: %w", err) + } + + err = common.WriteStringToFile(common.WriteStringToFileInput{ + Content: resp.String(), + Filename: f.Name(), + Mode: os.FileMode(0755), + }) + if err != nil { + return fmt.Errorf("Unable to write helm installer to file: %w", err) + } + + fi, err := os.Stat(f.Name()) + if err != nil { + return fmt.Errorf("Unable to get helm installer file size: %w", err) + } + + if fi.Size() == 0 { + return fmt.Errorf("Invalid helm installer filesize") + } + + common.LogInfo2Quiet("Running helm installer") + installerCmd, err := common.CallExecCommand(common.ExecCommandInput{ + Command: f.Name(), + StreamStdio: true, + }) + if err != nil { + return fmt.Errorf("Unable to call helm installer command: %w", err) + } + if installerCmd.ExitCode != 0 { + return fmt.Errorf("Invalid exit code from helm installer command: %d", installerCmd.ExitCode) + } + return nil } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 8b57e389a..388c59f02 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -33,6 +33,7 @@ var ( "letsencrypt-email-stag": true, "namespace": true, "network-interface": true, + "proxy": true, "rollback-on-failure": true, "token": true, } @@ -68,13 +69,13 @@ var KubernetesManifests = []Manifest{ } type HelmChart struct { - ReleaseName string ChartPath string - Namespace string CreateNamespace bool - Version string + Namespace string Path string + ReleaseName string RepoURL string + Version string } var HelmCharts = []HelmChart{ @@ -102,6 +103,14 @@ var HelmCharts = []HelmChart{ RepoURL: "https://helm.traefik.io/traefik", Version: "26.0.0", }, + { + ChartPath: "ingress-nginx", + CreateNamespace: true, + Namespace: "ingress-nginx", + ReleaseName: "ingress-nginx", + RepoURL: "https://kubernetes.github.io/ingress-nginx", + Version: "4.7.5", + }, } type HelmRepository struct { diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 38616f7d5..efd75328e 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -22,8 +22,9 @@ func main() { args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node") + proxy := args.String("proxy", "traefik", "proxy: proxy to use for all outbound traffic") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandInitialize(*serverIP, *taintScheduling) + err = scheduler_k3s.CommandInitialize(*proxy, *serverIP, *taintScheduling) case "cluster-add": args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError) allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index ac7867eaa..e37cb83fc 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -59,7 +59,11 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri } // CommandInitialize initializes a k3s cluster on the local server -func CommandInitialize(serverIP string, taintScheduling bool) error { +func CommandInitialize(proxy string, serverIP string, taintScheduling bool) error { + if proxy != "nginx" && proxy != "traefik" { + return fmt.Errorf("Invalid proxy: %s", proxy) + } + if err := isK3sInstalled(); err == nil { return fmt.Errorf("k3s already installed, cannot re-initialize k3s") } @@ -219,6 +223,11 @@ func CommandInitialize(serverIP string, taintScheduling bool) error { args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule") } + common.CommandPropertySet("scheduler-k3s", "--global", "proxy", proxy, DefaultProperties, GlobalProperties) + if proxy == "nginx" { + args = append(args, "--disable", "traefik") + } + common.LogInfo2Quiet("Running k3s installer") installerCmd, err := common.CallExecCommand(common.ExecCommandInput{ Command: f.Name(), @@ -298,7 +307,17 @@ func CommandInitialize(serverIP string, taintScheduling bool) error { } common.LogInfo2Quiet("Installing helm charts") - err = installHelmCharts(ctx, clientset) + err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool { + if chart.ChartPath == "traefik" && proxy == "nginx" { + return false + } + + if chart.ChartPath == "nginx" && proxy == "traefik" { + return false + } + + return true + }) if err != nil { return fmt.Errorf("Unable to install helm charts: %w", err) } diff --git a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml new file mode 100644 index 000000000..a2a49a659 --- /dev/null +++ b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml @@ -0,0 +1,45 @@ +controller: + metrics: + enabled: true + config: + # Print access log to file instead of stdout + # Separating acces logs from the rest + access-log-path: "/data/access.log" + log-format-escape-json: "true" + + extraVolumeMounts: + - name: data + mountPath: /data + extraVolumes: + - name: data + emptyDir: {} + extraContainers: + - name: stream-accesslog + image: busybox + args: + - /bin/sh + - -c + - tail -n+1 -F /data/access.log + imagePullPolicy: Always + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data + name: data + allowSnippetAnnotations: true + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scheme: http + prometheus.io/scrape: "true" + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + priorityClassName: "system-cluster-critical" From b35d6df67c87484973f95f4b85d907b9efc138bc Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 30 Jan 2024 10:56:42 -0500 Subject: [PATCH 02/12] feat: add support for displaying error logs --- .../templates/helm-config/ingress-nginx.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml index a2a49a659..54eb2a67f 100644 --- a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml +++ b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml @@ -5,8 +5,8 @@ controller: # Print access log to file instead of stdout # Separating acces logs from the rest access-log-path: "/data/access.log" + error-log-path: "/data/error.log" log-format-escape-json: "true" - extraVolumeMounts: - name: data mountPath: /data @@ -27,6 +27,20 @@ controller: volumeMounts: - mountPath: /data name: data + extraContainers: + - name: stream-errorlog + image: busybox + args: + - /bin/sh + - -c + - tail -n+1 -F /data/error.log + imagePullPolicy: Always + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data + name: data allowSnippetAnnotations: true podAnnotations: prometheus.io/path: /metrics From 5674a29211ed4fb1edc5b02f7493a41d9736aed5 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 30 Jan 2024 10:57:24 -0500 Subject: [PATCH 03/12] feat: add ability to use ingress-nginx for routing instead of traefik --- plugins/scheduler-k3s/functions.go | 4 ++ plugins/scheduler-k3s/report.go | 23 +++++++---- plugins/scheduler-k3s/scheduler_k3s.go | 9 ++-- .../src/subcommands/subcommands.go | 4 +- plugins/scheduler-k3s/subcommands.go | 14 +++---- .../templates/chart/ingress.yaml | 41 +++++++++++++++++++ plugins/scheduler-k3s/triggers.go | 4 +- 7 files changed, 74 insertions(+), 25 deletions(-) create mode 100644 plugins/scheduler-k3s/templates/chart/ingress.yaml diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index f2111ced7..9d39ead31 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -494,6 +494,10 @@ func getComputedImagePullSecrets(appName string) string { return imagePullSecrets } +func getGlobalIngressClass() string { + return common.PropertyGetDefault("scheduler-k3s", "global", "ingress-class", DefaultIngressClass) +} + func getLetsencryptServer(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "") } diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index 1974114fa..6636a506f 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -12,23 +12,24 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { flags := map[string]common.ReportFunc{ "--scheduler-k3s-computed-deploy-timeout": reportComputedDeployTimeout, + "--scheduler-k3s-computed-image-pull-secrets": reportComputedImagePullSecrets, + "--scheduler-k3s-computed-letsencrypt-server": reportComputedLetsencryptServer, + "--scheduler-k3s-computed-namespace": reportComputedNamespace, + "--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure, "--scheduler-k3s-deploy-timeout": reportDeployTimeout, "--scheduler-k3s-global-deploy-timeout": reportGlobalDeployTimeout, - "--scheduler-k3s-computed-image-pull-secrets": reportComputedImagePullSecrets, - "--scheduler-k3s-image-pull-secrets": reportImagePullSecrets, "--scheduler-k3s-global-image-pull-secrets": reportGlobalImagePullSecrets, - "--scheduler-k3s-computed-letsencrypt-server": reportComputedLetsencryptServer, - "--scheduler-k3s-letsencrypt-server": reportLetsencryptServer, - "--scheduler-k3s-global-letsencrypt-server": reportGlobalLetsencryptServer, + "--scheduler-k3s-global-ingress-class": reportGlobalIngressClass, "--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd, "--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag, - "--scheduler-k3s-computed-namespace": reportComputedNamespace, - "--scheduler-k3s-namespace": reportNamespace, + "--scheduler-k3s-global-letsencrypt-server": reportGlobalLetsencryptServer, "--scheduler-k3s-global-namespace": reportGlobalNamespace, "--scheduler-k3s-global-network-interface": reportGlobalNetworkInterface, - "--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure, - "--scheduler-k3s-rollback-on-failure": reportRollbackOnFailure, "--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure, + "--scheduler-k3s-image-pull-secrets": reportImagePullSecrets, + "--scheduler-k3s-letsencrypt-server": reportLetsencryptServer, + "--scheduler-k3s-namespace": reportNamespace, + "--scheduler-k3s-rollback-on-failure": reportRollbackOnFailure, } flagKeys := []string{} @@ -66,6 +67,10 @@ func reportGlobalImagePullSecrets(appName string) string { return getGlobalImagePullSecrets() } +func reportGlobalIngressClass(appName string) string { + return getGlobalIngressClass() +} + func reportComputedLetsencryptServer(appName string) string { return getComputedLetsencryptServer(appName) } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 388c59f02..78a857fa4 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -28,22 +28,21 @@ var ( GlobalProperties = map[string]bool{ "deploy-timeout": true, "image-pull-secrets": true, + "ingress-class": true, "letsencrypt-server": true, "letsencrypt-email-prod": true, "letsencrypt-email-stag": true, "namespace": true, "network-interface": true, - "proxy": true, "rollback-on-failure": true, "token": true, } ) -const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" - -const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml" - +const DefaultIngressClass = "traefik" const GlobalProcessType = "--global" +const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" +const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml" var ( runtimeScheme = runtime.NewScheme() diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index efd75328e..cf45bf252 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -22,9 +22,9 @@ func main() { args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node") - proxy := args.String("proxy", "traefik", "proxy: proxy to use for all outbound traffic") + ingressClass := args.String("ingress-class", "traefik", "ingress-class: ingress-class to use for all outbound traffic") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandInitialize(*proxy, *serverIP, *taintScheduling) + err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling) case "cluster-add": args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError) allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index e37cb83fc..5a72476d9 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -59,9 +59,9 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri } // CommandInitialize initializes a k3s cluster on the local server -func CommandInitialize(proxy string, serverIP string, taintScheduling bool) error { - if proxy != "nginx" && proxy != "traefik" { - return fmt.Errorf("Invalid proxy: %s", proxy) +func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error { + if ingressClass != "nginx" && ingressClass != "traefik" { + return fmt.Errorf("Invalid ingress-class: %s", ingressClass) } if err := isK3sInstalled(); err == nil { @@ -223,8 +223,8 @@ func CommandInitialize(proxy string, serverIP string, taintScheduling bool) erro args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule") } - common.CommandPropertySet("scheduler-k3s", "--global", "proxy", proxy, DefaultProperties, GlobalProperties) - if proxy == "nginx" { + common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties) + if ingressClass == "nginx" { args = append(args, "--disable", "traefik") } @@ -308,11 +308,11 @@ func CommandInitialize(proxy string, serverIP string, taintScheduling bool) erro common.LogInfo2Quiet("Installing helm charts") err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool { - if chart.ChartPath == "traefik" && proxy == "nginx" { + if chart.ChartPath == "traefik" && ingressClass == "nginx" { return false } - if chart.ChartPath == "nginx" && proxy == "traefik" { + if chart.ChartPath == "nginx" && ingressClass == "traefik" { return false } diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml new file mode 100644 index 000000000..afadc9275 --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -0,0 +1,41 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := .Values.processes.PROCESS_NAME }} +{{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} +{{- range $pdx, $port_map := $config.web.port_maps }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + dokku.com/managed: "true" + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }} + namespace: {{ $.Values.global.namespace }} +spec: + ingressClassName: nginx + {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }} + tls: + - hosts: + {{- range $ddx, $domain := $config.web.domains }} + - {{ $domain | quote }} + {{- end }} + secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }} + {{- end }} + rules: + {{- range $ddx, $domain := $config.web.domains }} + - host: {{ $domain | quote }} + http: + paths: + - backend: + service: + name: {{ $.Values.global.app_name }}-{{ $processName }} + port: + name: {{ $port_map.name }} + pathType: ImplementationSpecific + path: / + {{- end }} +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 723cf81b7..16f824271 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -351,7 +351,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e }, Namespace: namespace, Network: GlobalNetwork{ - IngressClass: "traefik", + IngressClass: common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass), PrimaryPort: primaryPort, }, Secrets: map[string]string{}, @@ -439,7 +439,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e templateFiles := []string{"deployment"} if processType == "web" { - templateFiles = append(templateFiles, "service", "certificate", "ingress-route", "https-redirect-middleware") + templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware") } for _, templateName := range templateFiles { b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) From 1af36b609d04cc893c9d28e15c6b4ad61ecafd42 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 5 Feb 2024 13:35:09 -0500 Subject: [PATCH 04/12] feat: add resource-specific annotations to ingress objects --- plugins/scheduler-k3s/template.go | 2 +- plugins/scheduler-k3s/templates/chart/ingress.yaml | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index b9c76c96a..c371daae8 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -73,8 +73,8 @@ type ProcessAnnotations struct { JobAnnotations map[string]string `yaml:"job,omitempty"` PodAnnotations map[string]string `yaml:"pod,omitempty"` SecretAnnotations map[string]string `yaml:"secret,omitempty"` - ServiceAnnotations map[string]string `yaml:"service,omitempty"` ServiceAccountAnnotations map[string]string `yaml:"serviceaccount,omitempty"` + ServiceAnnotations map[string]string `yaml:"service,omitempty"` TraefikIngressRouteAnnotations map[string]string `yaml:"traefik_ingressroute,omitempty"` TraefikMiddlewareAnnotations map[string]string `yaml:"traefik_middleware,omitempty"` } diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index afadc9275..25c9cd068 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,5 +1,5 @@ {{- $processName := "PROCESS_NAME" }} -{{- $config := .Values.processes.PROCESS_NAME }} +{{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} {{- range $pdx, $port_map := $config.web.port_maps }} --- @@ -8,6 +8,16 @@ kind: Ingress metadata: annotations: dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.deployment }} + {{- range $k, $v := $.Values.global.annotations.deployment }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.ingress }} + {{- range $k, $v := $config.annotations.ingress }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} From 416a119904d748625879ad377fb884e2488f2d6a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 5 Feb 2024 13:44:47 -0500 Subject: [PATCH 05/12] feat: pull nginx properties as ingress annotations --- plugins/scheduler-k3s/functions.go | 162 +++++++++++++++++++++++++++++ plugins/scheduler-k3s/go.mod | 5 +- plugins/scheduler-k3s/triggers.go | 2 + 3 files changed, 168 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 9d39ead31..a5b7101a3 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -14,6 +14,7 @@ import ( appjson "github.com/dokku/dokku/plugins/app-json" "github.com/dokku/dokku/plugins/common" + nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts" resty "github.com/go-resty/resty/v2" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" @@ -498,6 +499,167 @@ func getGlobalIngressClass() string { return common.PropertyGetDefault("scheduler-k3s", "global", "ingress-class", DefaultIngressClass) } +func getIngressAnnotations(appName string, processType string) map[string]string { + type annotation struct { + annotation string + getter func(appName string) string + locationSnippet func(value string) string + serverSnippet func(value string) string + } + + locationLines := []string{} + serverLines := []string{} + + properties := map[string]annotation{ + "access-log-path": { + getter: nginxvhosts.ComputedAccessLogPath, + serverSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("access_log %s;", value) + }, + }, + "bind-address-ipv4": { + getter: nginxvhosts.ComputedBindAddressIPv4, + }, + "bind-address-ipv6": { + getter: nginxvhosts.ComputedBindAddressIPv6, + }, + "client-max-body-size": { + annotation: "nginx.ingress.kubernetes.io/proxy-body-size", + getter: nginxvhosts.ComputedClientMaxBodySize, + }, + "disable-custom-config": { + getter: nginxvhosts.ComputedDisableCustomConfig, + }, + "error-log-path": { + getter: nginxvhosts.ComputedErrorLogPath, + serverSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("error_log %s;", value) + }, + }, + // todo: handle hsts properly + "hsts-include-subdomains": { + getter: nginxvhosts.ComputedHSTSIncludeSubdomains, + }, + "hsts-max-age": { + getter: nginxvhosts.ComputedHSTSMaxAge, + }, + "hsts-preload": { + getter: nginxvhosts.ComputedHSTSPreload, + }, + "hsts": { + getter: nginxvhosts.ComputedHSTS, + }, + "nginx-conf-sigil-path": { + getter: nginxvhosts.ComputedNginxConfSigilPath, + }, + "proxy-buffer-size": { + annotation: "nginx.ingress.kubernetes.io/proxy-buffer-size", + getter: nginxvhosts.ComputedProxyBufferSize, + }, + "proxy-buffering": { + getter: nginxvhosts.ComputedProxyBuffering, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_buffering %s;", value) + }, + }, + "proxy-buffers": { + annotation: "nginx.ingress.kubernetes.io/proxy-buffers-number", + getter: nginxvhosts.ComputedProxyBuffers, + }, + "proxy-busy-buffers-size": { + getter: nginxvhosts.ComputedProxyBusyBuffersSize, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_busy_buffers_size %s;", value) + }, + }, + "proxy-read-timeout": { + annotation: "nginx.ingress.kubernetes.io/proxy-read-timeout", + getter: nginxvhosts.ComputedProxyReadTimeout, + }, + "x-forwarded-for-value": { + getter: nginxvhosts.ComputedXForwardedForValue, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-For %s;", value) + }, + }, + "x-forwarded-port-value": { + getter: nginxvhosts.ComputedXForwardedPortValue, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-Port %s;", value) + }, + }, + "x-forwarded-proto-value": { + getter: nginxvhosts.ComputedXForwardedProtoValue, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-Proto %s;", value) + }, + }, + "x-forwarded-ssl": { + getter: nginxvhosts.ComputedXForwardedSSL, + locationSnippet: func(value string) string { + if value == "" { + return "" + } + return fmt.Sprintf("proxy_set_header X-Forwarded-SSL %s;", value) + }, + }, + } + + data := map[string]string{} + for _, newKey := range properties { + if newKey.locationSnippet != nil { + locationLines = append(locationLines, newKey.locationSnippet(newKey.getter(appName))) + } else if newKey.serverSnippet != nil { + serverLines = append(serverLines, newKey.serverSnippet(newKey.getter(appName))) + } else if newKey.annotation != "" { + data[newKey.annotation] = newKey.getter(appName) + } + } + + var locationSnippet string + for _, line := range locationLines { + if line != "" { + locationSnippet += line + "\n" + } + } + var serverSnippet string + for _, line := range serverLines { + if line != "" { + serverSnippet += line + "\n" + } + } + + if locationSnippet != "" { + data["nginx.ingress.kubernetes.io/configuration-snippet"] = locationSnippet + } + if serverSnippet != "" { + data["nginx.ingress.kubernetes.io/server-snippet"] = serverSnippet + } + + return data +} + func getLetsencryptServer(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "") } diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index ba0ea0ab4..9ef162d0d 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -8,6 +8,7 @@ require ( github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000 + github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-00010101000000-000000000000 github.com/fatih/color v1.16.0 github.com/go-openapi/jsonpointer v0.19.6 github.com/go-resty/resty/v2 v2.11.0 @@ -18,6 +19,7 @@ require ( github.com/ryanuber/columnize v2.1.2+incompatible github.com/spf13/pflag v1.0.5 github.com/traefik/traefik/v2 v2.10.7 + golang.org/x/sync v0.6.0 gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.13.3 k8s.io/api v0.29.0 @@ -152,7 +154,6 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -186,3 +187,5 @@ replace github.com/dokku/dokku/plugins/common => ../common replace github.com/dokku/dokku/plugins/config => ../config replace github.com/dokku/dokku/plugins/cron => ../cron + +replace github.com/dokku/dokku/plugins/nginx-vhosts => ../nginx-vhosts diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 16f824271..1591ecb03 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -410,6 +410,8 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e }, } + processValues.Annotations.IngressAnnotations = getIngressAnnotations(appName, processType) + processValues.ProcessType = ProcessType_Web for _, portMap := range portMaps { protocol := PortmapProtocol_TCP From 7dbd9adc3332036cbf7e7633cea622d6db1a2a3b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 6 Feb 2024 01:24:39 -0500 Subject: [PATCH 06/12] fix: set log mount dir to /var/log/nginx --- .../templates/helm-config/ingress-nginx.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml index 54eb2a67f..0e317922e 100644 --- a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml +++ b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml @@ -4,12 +4,12 @@ controller: config: # Print access log to file instead of stdout # Separating acces logs from the rest - access-log-path: "/data/access.log" - error-log-path: "/data/error.log" + access-log-path: "/var/log/nginx/access.log" + error-log-path: "/var/log/nginx/error.log" log-format-escape-json: "true" extraVolumeMounts: - name: data - mountPath: /data + mountPath: /var/log/nginx extraVolumes: - name: data emptyDir: {} @@ -19,13 +19,13 @@ controller: args: - /bin/sh - -c - - tail -n+1 -F /data/access.log + - tail -n+1 -F /var/log/nginx/access.log imagePullPolicy: Always resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /data + - mountPath: /var/log/nginx name: data extraContainers: - name: stream-errorlog @@ -33,13 +33,13 @@ controller: args: - /bin/sh - -c - - tail -n+1 -F /data/error.log + - tail -n+1 -F /var/log/nginx/error.log imagePullPolicy: Always resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /data + - mountPath: /var/log/nginx name: data allowSnippetAnnotations: true podAnnotations: From 980979ef874827515c2e037a9f914d355ff9f060 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 6 Feb 2024 22:48:22 -0500 Subject: [PATCH 07/12] feat: properly handle nginx annotation merging --- plugins/scheduler-k3s/functions.go | 27 ++++++++++++++++++++------- plugins/scheduler-k3s/triggers.go | 2 -- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index a5b7101a3..68b860df0 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -385,7 +385,7 @@ func getAnnotations(appName string, processType string) (ProcessAnnotations, err } annotations.DeploymentAnnotations = deploymentAnnotations - ingressAnnotations, err := getAnnotation(appName, processType, "ingress") + ingressAnnotations, err := getIngressAnnotations(appName, processType) if err != nil { return annotations, err } @@ -499,7 +499,7 @@ func getGlobalIngressClass() string { return common.PropertyGetDefault("scheduler-k3s", "global", "ingress-class", DefaultIngressClass) } -func getIngressAnnotations(appName string, processType string) map[string]string { +func getIngressAnnotations(appName string, processType string) (map[string]string, error) { type annotation struct { annotation string getter func(appName string) string @@ -626,14 +626,14 @@ func getIngressAnnotations(appName string, processType string) map[string]string }, } - data := map[string]string{} + annotations := map[string]string{} for _, newKey := range properties { if newKey.locationSnippet != nil { locationLines = append(locationLines, newKey.locationSnippet(newKey.getter(appName))) } else if newKey.serverSnippet != nil { serverLines = append(serverLines, newKey.serverSnippet(newKey.getter(appName))) } else if newKey.annotation != "" { - data[newKey.annotation] = newKey.getter(appName) + annotations[newKey.annotation] = newKey.getter(appName) } } @@ -651,13 +651,26 @@ func getIngressAnnotations(appName string, processType string) map[string]string } if locationSnippet != "" { - data["nginx.ingress.kubernetes.io/configuration-snippet"] = locationSnippet + annotations["nginx.ingress.kubernetes.io/configuration-snippet"] = locationSnippet } if serverSnippet != "" { - data["nginx.ingress.kubernetes.io/server-snippet"] = serverSnippet + annotations["nginx.ingress.kubernetes.io/server-snippet"] = serverSnippet } - return data + customAnnotations, err := getAnnotation(appName, processType, "deployment") + if err != nil { + return map[string]string{}, err + } + + for key, value := range customAnnotations { + if _, ok := annotations[key]; ok { + common.LogWarn(fmt.Sprintf("Nginx-based annotation %s will be overwritten by custom annotation", key)) + } + + annotations[key] = value + } + + return annotations, nil } func getLetsencryptServer(appName string) string { diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 1591ecb03..16f824271 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -410,8 +410,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e }, } - processValues.Annotations.IngressAnnotations = getIngressAnnotations(appName, processType) - processValues.ProcessType = ProcessType_Web for _, portMap := range portMaps { protocol := PortmapProtocol_TCP From e9986b5775e5bb0740533c5a3e8d0d60befaf799 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 6 Feb 2024 23:00:32 -0500 Subject: [PATCH 08/12] chore: unsort flags --- plugins/scheduler-k3s/report.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index 6636a506f..25474883b 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -12,24 +12,24 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { flags := map[string]common.ReportFunc{ "--scheduler-k3s-computed-deploy-timeout": reportComputedDeployTimeout, - "--scheduler-k3s-computed-image-pull-secrets": reportComputedImagePullSecrets, - "--scheduler-k3s-computed-letsencrypt-server": reportComputedLetsencryptServer, - "--scheduler-k3s-computed-namespace": reportComputedNamespace, - "--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure, "--scheduler-k3s-deploy-timeout": reportDeployTimeout, "--scheduler-k3s-global-deploy-timeout": reportGlobalDeployTimeout, + "--scheduler-k3s-computed-image-pull-secrets": reportComputedImagePullSecrets, + "--scheduler-k3s-image-pull-secrets": reportImagePullSecrets, "--scheduler-k3s-global-image-pull-secrets": reportGlobalImagePullSecrets, + "--scheduler-k3s-computed-letsencrypt-server": reportComputedLetsencryptServer, + "--scheduler-k3s-letsencrypt-server": reportLetsencryptServer, + "--scheduler-k3s-global-letsencrypt-server": reportGlobalLetsencryptServer, "--scheduler-k3s-global-ingress-class": reportGlobalIngressClass, "--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd, "--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag, - "--scheduler-k3s-global-letsencrypt-server": reportGlobalLetsencryptServer, + "--scheduler-k3s-computed-namespace": reportComputedNamespace, + "--scheduler-k3s-namespace": reportNamespace, "--scheduler-k3s-global-namespace": reportGlobalNamespace, "--scheduler-k3s-global-network-interface": reportGlobalNetworkInterface, - "--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure, - "--scheduler-k3s-image-pull-secrets": reportImagePullSecrets, - "--scheduler-k3s-letsencrypt-server": reportLetsencryptServer, - "--scheduler-k3s-namespace": reportNamespace, + "--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure, "--scheduler-k3s-rollback-on-failure": reportRollbackOnFailure, + "--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure, } flagKeys := []string{} From 03ae424a274c7de491de4cb31ad2649c071db444 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 7 Feb 2024 00:01:57 -0500 Subject: [PATCH 09/12] fix: ensure map keys are unique --- .../templates/helm-config/ingress-nginx.yaml | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml index 0e317922e..8263f6d42 100644 --- a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml +++ b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml @@ -1,11 +1,10 @@ +--- controller: metrics: enabled: true config: - # Print access log to file instead of stdout - # Separating acces logs from the rest - access-log-path: "/var/log/nginx/access.log" - error-log-path: "/var/log/nginx/error.log" + access-log-path: /var/log/nginx/access.log + error-log-path: /var/log/nginx/error.log log-format-escape-json: "true" extraVolumeMounts: - name: data @@ -17,30 +16,29 @@ controller: - name: stream-accesslog image: busybox args: - - /bin/sh - - -c - - tail -n+1 -F /var/log/nginx/access.log + - /bin/sh + - -c + - tail -n+1 -F /var/log/nginx/access.log imagePullPolicy: Always resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /var/log/nginx - name: data - extraContainers: + - mountPath: /var/log/nginx + name: data - name: stream-errorlog image: busybox args: - - /bin/sh - - -c - - tail -n+1 -F /var/log/nginx/error.log + - /bin/sh + - -c + - tail -n+1 -F /var/log/nginx/error.log imagePullPolicy: Always resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /var/log/nginx - name: data + - mountPath: /var/log/nginx + name: data allowSnippetAnnotations: true podAnnotations: prometheus.io/path: /metrics @@ -48,12 +46,12 @@ controller: prometheus.io/scheme: http prometheus.io/scrape: "true" tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - - key: "node-role.kubernetes.io/control-plane" - operator: "Exists" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" - priorityClassName: "system-cluster-critical" + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + priorityClassName: system-cluster-critical From afb4b50bb3c1e96531c5b8dac203c471cf6affcf Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 7 Feb 2024 02:20:37 -0500 Subject: [PATCH 10/12] docs: document how to switch to nginx --- docs/deployment/schedulers/k3s.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 6054812b8..540bd157d 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -55,6 +55,14 @@ By default, Dokku will attempt to auto-detect the IP address of the server. In c dokku scheduler-k3s:initialize --server-ip 192.168.20.15 ``` +Dokku's k3s integration natively uses `Traefik` as it's ingress load balancer via [Traefik's CRDs](https://doc.traefik.io/traefik/providers/kubernetes-crd/), but a cluster can be set to use `nginx` via the [ingress-nginx](https://github.com/kubernetes/ingress-nginx) project. Switching to nginx will result in any `nginx` plugin settings being respected, either by turning them into annotations or creating a custom server snippet. + +To change the ingress, set the `--ingress-class` flag: + +```shell +dokku scheduler-k3s:initialize --ingress-class nginx +``` + ### Adding nodes to the cluster > [!WARNING] From 41a89bbb5db73ea5ba43a58b834ddc6defe8d85f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 7 Feb 2024 02:20:49 -0500 Subject: [PATCH 11/12] feat: add a test for ingress-nginx support --- tests/unit/scheduler-k3s.bats | 37 ++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/unit/scheduler-k3s.bats b/tests/unit/scheduler-k3s.bats index ca503a258..182f489e7 100644 --- a/tests/unit/scheduler-k3s.bats +++ b/tests/unit/scheduler-k3s.bats @@ -49,7 +49,8 @@ install_k3s() { assert_success assert_output_contains "Login Succeeded" - run /bin/bash -c "dokku scheduler-k3s:initialize" + INGRESS_CLASS="${INGRESS_CLASS:-traefik}" + run /bin/bash -c "dokku scheduler-k3s:initialize --ingress-class $INGRESS_CLASS" echo "output: $output" echo "status: $status" assert_success @@ -70,12 +71,42 @@ uninstall_k3s() { install_k3s } -@test "(scheduler-k3s) deploy" { +@test "(scheduler-k3s) deploy traefik" { if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" fi - install_k3s + INGRESS_CLASS=traefik install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 20" + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + +@test "(scheduler-k3s) deploy nginx" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s run /bin/bash -c "dokku apps:create $TEST_APP" echo "output: $output" From cd4f9db95bd30cb008c6bcbf729d0e85c0b47679 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 7 Feb 2024 13:15:16 -0500 Subject: [PATCH 12/12] fix: use annotation instead of location snippet --- plugins/scheduler-k3s/functions.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 68b860df0..5571cb19e 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -563,13 +563,8 @@ func getIngressAnnotations(appName string, processType string) (map[string]strin getter: nginxvhosts.ComputedProxyBufferSize, }, "proxy-buffering": { - getter: nginxvhosts.ComputedProxyBuffering, - locationSnippet: func(value string) string { - if value == "" { - return "" - } - return fmt.Sprintf("proxy_buffering %s;", value) - }, + annotation: "nginx.ingress.kubernetes.io/proxy-buffering", + getter: nginxvhosts.ComputedProxyBuffering, }, "proxy-buffers": { annotation: "nginx.ingress.kubernetes.io/proxy-buffers-number",