From 81286a067362ed6232d65510d9b700dc083da998 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 30 Jan 2024 10:20:12 -0500 Subject: [PATCH] feat: add ability to install nginx instead of traefik --- plugins/scheduler-k3s/functions.go | 65 ++++++++++++++++++- plugins/scheduler-k3s/scheduler_k3s.go | 15 ++++- .../src/subcommands/subcommands.go | 3 +- plugins/scheduler-k3s/subcommands.go | 23 ++++++- .../templates/helm-config/ingress-nginx.yaml | 45 +++++++++++++ 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index c783d4502..f2111ced7 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -754,7 +754,7 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { }, nil } -func installHelmCharts(ctx context.Context, clientset KubernetesClient) error { +func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error { for _, repo := range HelmRepositories { helmAgent, err := NewHelmAgent("default", DeployLogPrinter) if err != nil { @@ -768,6 +768,10 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error { } for _, chart := range HelmCharts { + if !shouldInstall(chart) { + continue + } + if chart.CreateNamespace { namespace := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -873,6 +877,65 @@ func installHelperCommands(ctx context.Context) error { } } + return installHelm(ctx) +} + +func installHelm(ctx context.Context) error { + client := resty.New() + resp, err := client.R(). + SetContext(ctx). + Get("https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3") + if err != nil { + return fmt.Errorf("Unable to download helm installer: %w", err) + } + if resp == nil { + return fmt.Errorf("Missing response from helm installer download: %w", err) + } + + if resp.StatusCode() != 200 { + return fmt.Errorf("Invalid status code for helm installer script: %d", resp.StatusCode()) + } + + f, err := os.CreateTemp("", "sample") + if err != nil { + return fmt.Errorf("Unable to create temporary file for helm installer: %w", err) + } + defer os.Remove(f.Name()) + + if err := f.Close(); err != nil { + return fmt.Errorf("Unable to close helm installer file: %w", err) + } + + err = common.WriteStringToFile(common.WriteStringToFileInput{ + Content: resp.String(), + Filename: f.Name(), + Mode: os.FileMode(0755), + }) + if err != nil { + return fmt.Errorf("Unable to write helm installer to file: %w", err) + } + + fi, err := os.Stat(f.Name()) + if err != nil { + return fmt.Errorf("Unable to get helm installer file size: %w", err) + } + + if fi.Size() == 0 { + return fmt.Errorf("Invalid helm installer filesize") + } + + common.LogInfo2Quiet("Running helm installer") + installerCmd, err := common.CallExecCommand(common.ExecCommandInput{ + Command: f.Name(), + StreamStdio: true, + }) + if err != nil { + return fmt.Errorf("Unable to call helm installer command: %w", err) + } + if installerCmd.ExitCode != 0 { + return fmt.Errorf("Invalid exit code from helm installer command: %d", installerCmd.ExitCode) + } + return nil } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 8b57e389a..388c59f02 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -33,6 +33,7 @@ var ( "letsencrypt-email-stag": true, "namespace": true, "network-interface": true, + "proxy": true, "rollback-on-failure": true, "token": true, } @@ -68,13 +69,13 @@ var KubernetesManifests = []Manifest{ } type HelmChart struct { - ReleaseName string ChartPath string - Namespace string CreateNamespace bool - Version string + Namespace string Path string + ReleaseName string RepoURL string + Version string } var HelmCharts = []HelmChart{ @@ -102,6 +103,14 @@ var HelmCharts = []HelmChart{ RepoURL: "https://helm.traefik.io/traefik", Version: "26.0.0", }, + { + ChartPath: "ingress-nginx", + CreateNamespace: true, + Namespace: "ingress-nginx", + ReleaseName: "ingress-nginx", + RepoURL: "https://kubernetes.github.io/ingress-nginx", + Version: "4.7.5", + }, } type HelmRepository struct { diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 38616f7d5..efd75328e 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -22,8 +22,9 @@ func main() { args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node") + proxy := args.String("proxy", "traefik", "proxy: proxy to use for all outbound traffic") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandInitialize(*serverIP, *taintScheduling) + err = scheduler_k3s.CommandInitialize(*proxy, *serverIP, *taintScheduling) case "cluster-add": args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError) allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index ac7867eaa..e37cb83fc 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -59,7 +59,11 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri } // CommandInitialize initializes a k3s cluster on the local server -func CommandInitialize(serverIP string, taintScheduling bool) error { +func CommandInitialize(proxy string, serverIP string, taintScheduling bool) error { + if proxy != "nginx" && proxy != "traefik" { + return fmt.Errorf("Invalid proxy: %s", proxy) + } + if err := isK3sInstalled(); err == nil { return fmt.Errorf("k3s already installed, cannot re-initialize k3s") } @@ -219,6 +223,11 @@ func CommandInitialize(serverIP string, taintScheduling bool) error { args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule") } + common.CommandPropertySet("scheduler-k3s", "--global", "proxy", proxy, DefaultProperties, GlobalProperties) + if proxy == "nginx" { + args = append(args, "--disable", "traefik") + } + common.LogInfo2Quiet("Running k3s installer") installerCmd, err := common.CallExecCommand(common.ExecCommandInput{ Command: f.Name(), @@ -298,7 +307,17 @@ func CommandInitialize(serverIP string, taintScheduling bool) error { } common.LogInfo2Quiet("Installing helm charts") - err = installHelmCharts(ctx, clientset) + err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool { + if chart.ChartPath == "traefik" && proxy == "nginx" { + return false + } + + if chart.ChartPath == "nginx" && proxy == "traefik" { + return false + } + + return true + }) if err != nil { return fmt.Errorf("Unable to install helm charts: %w", err) } diff --git a/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml new file mode 100644 index 000000000..a2a49a659 --- /dev/null +++ b/plugins/scheduler-k3s/templates/helm-config/ingress-nginx.yaml @@ -0,0 +1,45 @@ +controller: + metrics: + enabled: true + config: + # Print access log to file instead of stdout + # Separating acces logs from the rest + access-log-path: "/data/access.log" + log-format-escape-json: "true" + + extraVolumeMounts: + - name: data + mountPath: /data + extraVolumes: + - name: data + emptyDir: {} + extraContainers: + - name: stream-accesslog + image: busybox + args: + - /bin/sh + - -c + - tail -n+1 -F /data/access.log + imagePullPolicy: Always + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data + name: data + allowSnippetAnnotations: true + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scheme: http + prometheus.io/scrape: "true" + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + priorityClassName: "system-cluster-critical"