feat: add ability to install nginx instead of traefik

plugins/scheduler-k3s/functions.go

@@ -754,7 +754,7 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
-func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
+func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error {
 	for _, repo := range HelmRepositories {
 		helmAgent, err := NewHelmAgent("default", DeployLogPrinter)
 		if err != nil {
@@ -768,6 +768,10 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
 	}
 
 	for _, chart := range HelmCharts {
+		if !shouldInstall(chart) {
+			continue
+		}
+
 		if chart.CreateNamespace {
 			namespace := corev1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{
@@ -873,6 +877,65 @@ func installHelperCommands(ctx context.Context) error {
 		}
 	}
 
+	return installHelm(ctx)
+}
+
+func installHelm(ctx context.Context) error {
+	client := resty.New()
+	resp, err := client.R().
+		SetContext(ctx).
+		Get("https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3")
+	if err != nil {
+		return fmt.Errorf("Unable to download helm installer: %w", err)
+	}
+	if resp == nil {
+		return fmt.Errorf("Missing response from helm installer download: %w", err)
+	}
+
+	if resp.StatusCode() != 200 {
+		return fmt.Errorf("Invalid status code for helm installer script: %d", resp.StatusCode())
+	}
+
+	f, err := os.CreateTemp("", "sample")
+	if err != nil {
+		return fmt.Errorf("Unable to create temporary file for helm installer: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("Unable to close helm installer file: %w", err)
+	}
+
+	err = common.WriteStringToFile(common.WriteStringToFileInput{
+		Content:  resp.String(),
+		Filename: f.Name(),
+		Mode:     os.FileMode(0755),
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to write helm installer to file: %w", err)
+	}
+
+	fi, err := os.Stat(f.Name())
+	if err != nil {
+		return fmt.Errorf("Unable to get helm installer file size: %w", err)
+	}
+
+	if fi.Size() == 0 {
+		return fmt.Errorf("Invalid helm installer filesize")
+	}
+
+	common.LogInfo2Quiet("Running helm installer")
+	installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command:     f.Name(),
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call helm installer command: %w", err)
+	}
+	if installerCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from helm installer command: %d", installerCmd.ExitCode)
+	}
+
 	return nil
 }
 

plugins/scheduler-k3s/scheduler_k3s.go

@@ -33,6 +33,7 @@ var (
 		"letsencrypt-email-stag": true,
 		"namespace":              true,
 		"network-interface":      true,
+		"proxy":                  true,
 		"rollback-on-failure":    true,
 		"token":                  true,
 	}
@@ -68,13 +69,13 @@ var KubernetesManifests = []Manifest{
 }
 
 type HelmChart struct {
-	ReleaseName     string
 	ChartPath       string
-	Namespace       string
 	CreateNamespace bool
-	Version         string
+	Namespace       string
 	Path            string
+	ReleaseName     string
 	RepoURL         string
+	Version         string
 }
 
 var HelmCharts = []HelmChart{
@@ -102,6 +103,14 @@ var HelmCharts = []HelmChart{
 		RepoURL:         "https://helm.traefik.io/traefik",
 		Version:         "26.0.0",
 	},
+	{
+		ChartPath:       "ingress-nginx",
+		CreateNamespace: true,
+		Namespace:       "ingress-nginx",
+		ReleaseName:     "ingress-nginx",
+		RepoURL:         "https://kubernetes.github.io/ingress-nginx",
+		Version:         "4.7.5",
+	},
 }
 
 type HelmRepository struct {

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -22,8 +22,9 @@ func main() {
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
 		serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
+		proxy := args.String("proxy", "traefik", "proxy: proxy to use for all outbound traffic")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandInitialize(*serverIP, *taintScheduling)
+		err = scheduler_k3s.CommandInitialize(*proxy, *serverIP, *taintScheduling)
 	case "cluster-add":
 		args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError)
 		allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts")

plugins/scheduler-k3s/subcommands.go

@@ -59,7 +59,11 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri
 }
 
 // CommandInitialize initializes a k3s cluster on the local server
-func CommandInitialize(serverIP string, taintScheduling bool) error {
+func CommandInitialize(proxy string, serverIP string, taintScheduling bool) error {
+	if proxy != "nginx" && proxy != "traefik" {
+		return fmt.Errorf("Invalid proxy: %s", proxy)
+	}
+
 	if err := isK3sInstalled(); err == nil {
 		return fmt.Errorf("k3s already installed, cannot re-initialize k3s")
 	}
@@ -219,6 +223,11 @@ func CommandInitialize(serverIP string, taintScheduling bool) error {
 		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
 	}
 
+	common.CommandPropertySet("scheduler-k3s", "--global", "proxy", proxy, DefaultProperties, GlobalProperties)
+	if proxy == "nginx" {
+		args = append(args, "--disable", "traefik")
+	}
+
 	common.LogInfo2Quiet("Running k3s installer")
 	installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
 		Command:     f.Name(),
@@ -298,7 +307,17 @@ func CommandInitialize(serverIP string, taintScheduling bool) error {
 	}
 
 	common.LogInfo2Quiet("Installing helm charts")
-	err = installHelmCharts(ctx, clientset)
+	err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool {
+		if chart.ChartPath == "traefik" && proxy == "nginx" {
+			return false
+		}
+
+		if chart.ChartPath == "nginx" && proxy == "traefik" {
+			return false
+		}
+
+		return true
+	})
 	if err != nil {
 		return fmt.Errorf("Unable to install helm charts: %w", err)
 	}

plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml

@@ -0,0 +1,45 @@
+controller:
+  metrics:
+    enabled: true
+  config:
+    # Print access log to file instead of stdout
+    # Separating acces logs from the rest
+    access-log-path: "/data/access.log"
+    log-format-escape-json: "true"
+
+  extraVolumeMounts:
+    - name: data
+      mountPath: /data
+  extraVolumes:
+    - name: data
+      emptyDir: {}
+  extraContainers:
+    - name: stream-accesslog
+      image: busybox
+      args:
+      - /bin/sh
+      - -c
+      - tail -n+1 -F /data/access.log
+      imagePullPolicy: Always
+      resources: {}
+      terminationMessagePath: /dev/termination-log
+      terminationMessagePolicy: File
+      volumeMounts:
+      - mountPath: /data
+        name: data
+  allowSnippetAnnotations: true
+  podAnnotations:
+    prometheus.io/path: /metrics
+    prometheus.io/port: "10254"
+    prometheus.io/scheme: http
+    prometheus.io/scrape: "true"
+  tolerations:
+  - key: "CriticalAddonsOnly"
+    operator: "Exists"
+  - key: "node-role.kubernetes.io/control-plane"
+    operator: "Exists"
+    effect: "NoSchedule"
+  - key: "node-role.kubernetes.io/master"
+    operator: "Exists"
+    effect: "NoSchedule"
+  priorityClassName: "system-cluster-critical"