feat: add ability to use ingress-nginx for routing instead of traefik

2024-01-30 10:57:24 -05:00
parent b35d6df67c
commit 5674a29211
7 changed files with 74 additions and 25 deletions
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -494,6 +494,10 @@ func getComputedImagePullSecrets(appName string) string {
 	return imagePullSecrets
 }

+func getGlobalIngressClass() string {
+	return common.PropertyGetDefault("scheduler-k3s", "global", "ingress-class", DefaultIngressClass)
+}
+
 func getLetsencryptServer(appName string) string {
 	return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "")
 }
--- a/plugins/scheduler-k3s/report.go
+++ b/plugins/scheduler-k3s/report.go
@@ -12,23 +12,24 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {

 	flags := map[string]common.ReportFunc{
 		"--scheduler-k3s-computed-deploy-timeout":       reportComputedDeployTimeout,
+		"--scheduler-k3s-computed-image-pull-secrets":   reportComputedImagePullSecrets,
+		"--scheduler-k3s-computed-letsencrypt-server":   reportComputedLetsencryptServer,
+		"--scheduler-k3s-computed-namespace":            reportComputedNamespace,
+		"--scheduler-k3s-computed-rollback-on-failure":  reportComputedRollbackOnFailure,
 		"--scheduler-k3s-deploy-timeout":                reportDeployTimeout,
 		"--scheduler-k3s-global-deploy-timeout":         reportGlobalDeployTimeout,
-		"--scheduler-k3s-computed-image-pull-secrets":   reportComputedImagePullSecrets,
-		"--scheduler-k3s-image-pull-secrets":            reportImagePullSecrets,
 		"--scheduler-k3s-global-image-pull-secrets":     reportGlobalImagePullSecrets,
-		"--scheduler-k3s-computed-letsencrypt-server":   reportComputedLetsencryptServer,
-		"--scheduler-k3s-letsencrypt-server":            reportLetsencryptServer,
-		"--scheduler-k3s-global-letsencrypt-server":     reportGlobalLetsencryptServer,
+		"--scheduler-k3s-global-ingress-class":          reportGlobalIngressClass,
 		"--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd,
 		"--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag,
-		"--scheduler-k3s-computed-namespace":            reportComputedNamespace,
-		"--scheduler-k3s-namespace":                     reportNamespace,
+		"--scheduler-k3s-global-letsencrypt-server":     reportGlobalLetsencryptServer,
 		"--scheduler-k3s-global-namespace":              reportGlobalNamespace,
 		"--scheduler-k3s-global-network-interface":      reportGlobalNetworkInterface,
-		"--scheduler-k3s-computed-rollback-on-failure":  reportComputedRollbackOnFailure,
-		"--scheduler-k3s-rollback-on-failure":           reportRollbackOnFailure,
 		"--scheduler-k3s-global-rollback-on-failure":    reportGlobalRollbackOnFailure,
+		"--scheduler-k3s-image-pull-secrets":            reportImagePullSecrets,
+		"--scheduler-k3s-letsencrypt-server":            reportLetsencryptServer,
+		"--scheduler-k3s-namespace":                     reportNamespace,
+		"--scheduler-k3s-rollback-on-failure":           reportRollbackOnFailure,
 	}

 	flagKeys := []string{}
@@ -66,6 +67,10 @@ func reportGlobalImagePullSecrets(appName string) string {
 	return getGlobalImagePullSecrets()
 }

+func reportGlobalIngressClass(appName string) string {
+	return getGlobalIngressClass()
+}
+
 func reportComputedLetsencryptServer(appName string) string {
 	return getComputedLetsencryptServer(appName)
 }
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -28,22 +28,21 @@ var (
 	GlobalProperties = map[string]bool{
 		"deploy-timeout":         true,
 		"image-pull-secrets":     true,
+		"ingress-class":          true,
 		"letsencrypt-server":     true,
 		"letsencrypt-email-prod": true,
 		"letsencrypt-email-stag": true,
 		"namespace":              true,
 		"network-interface":      true,
-		"proxy":                  true,
 		"rollback-on-failure":    true,
 		"token":                  true,
 	}
 )

-const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
-
-const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml"
-
+const DefaultIngressClass = "traefik"
 const GlobalProcessType = "--global"
+const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
+const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml"

 var (
 	runtimeScheme  = runtime.NewScheme()
--- a/plugins/scheduler-k3s/src/subcommands/subcommands.go
+++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go
@@ -22,9 +22,9 @@ func main() {
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
 		serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
-		proxy := args.String("proxy", "traefik", "proxy: proxy to use for all outbound traffic")
+		ingressClass := args.String("ingress-class", "traefik", "ingress-class: ingress-class to use for all outbound traffic")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandInitialize(*proxy, *serverIP, *taintScheduling)
+		err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling)
 	case "cluster-add":
 		args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError)
 		allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts")
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -59,9 +59,9 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri
 }

 // CommandInitialize initializes a k3s cluster on the local server
-func CommandInitialize(proxy string, serverIP string, taintScheduling bool) error {
-	if proxy != "nginx" && proxy != "traefik" {
-		return fmt.Errorf("Invalid proxy: %s", proxy)
+func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error {
+	if ingressClass != "nginx" && ingressClass != "traefik" {
+		return fmt.Errorf("Invalid ingress-class: %s", ingressClass)
 	}

 	if err := isK3sInstalled(); err == nil {
@@ -223,8 +223,8 @@ func CommandInitialize(proxy string, serverIP string, taintScheduling bool) erro
 		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
 	}

-	common.CommandPropertySet("scheduler-k3s", "--global", "proxy", proxy, DefaultProperties, GlobalProperties)
-	if proxy == "nginx" {
+	common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties)
+	if ingressClass == "nginx" {
 		args = append(args, "--disable", "traefik")
 	}

@@ -308,11 +308,11 @@ func CommandInitialize(proxy string, serverIP string, taintScheduling bool) erro

 	common.LogInfo2Quiet("Installing helm charts")
 	err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool {
-		if chart.ChartPath == "traefik" && proxy == "nginx" {
+		if chart.ChartPath == "traefik" && ingressClass == "nginx" {
 			return false
 		}

-		if chart.ChartPath == "nginx" && proxy == "traefik" {
+		if chart.ChartPath == "nginx" && ingressClass == "traefik" {
 			return false
 		}

--- a/plugins/scheduler-k3s/templates/chart/ingress.yaml
+++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml
@@ -0,0 +1,41 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := .Values.processes.PROCESS_NAME }}
+{{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }}
+{{- range $pdx, $port_map := $config.web.port_maps }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }}
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  ingressClassName: nginx
+  {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }}
+  tls:
+    - hosts:
+    {{- range $ddx, $domain := $config.web.domains }}
+      - {{ $domain | quote }}
+    {{- end }}
+    secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
+  {{- end }}
+  rules:
+    {{- range $ddx, $domain := $config.web.domains }}
+    - host: {{ $domain | quote }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: {{ $.Values.global.app_name }}-{{ $processName }}
+                port:
+                  name: {{ $port_map.name }}
+            pathType: ImplementationSpecific
+            path: /
+    {{- end }}
+{{- end }}
+{{- end }}
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -351,7 +351,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			},
 			Namespace: namespace,
 			Network: GlobalNetwork{
-				IngressClass: "traefik",
+				IngressClass: common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass),
 				PrimaryPort:  primaryPort,
 			},
 			Secrets: map[string]string{},
@@ -439,7 +439,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e

 		templateFiles := []string{"deployment"}
 		if processType == "web" {
-			templateFiles = append(templateFiles, "service", "certificate", "ingress-route", "https-redirect-middleware")
+			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware")
 		}
 		for _, templateName := range templateFiles {
 			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))