refactor: install traefik via helm chart directly to avoid issues where traefik silently fails installation

plugins/scheduler-k3s/scheduler_k3s.go

@@ -92,6 +92,14 @@ var HelmCharts = []HelmChart{
 		RepoURL:         "https://charts.longhorn.io",
 		Version:         "1.5.3",
 	},
+	{
+		ChartPath:       "traefik",
+		CreateNamespace: true,
+		Namespace:       "traefik",
+		ReleaseName:     "traefik",
+		RepoURL:         "https://helm.traefik.io/traefik",
+		Version:         "26.0.0",
+	},
 }
 
 type HelmRepository struct {
@@ -108,6 +116,10 @@ var HelmRepositories = []HelmRepository{
 		Name: "longhorn",
 		URL:  "https://charts.longhorn.io",
 	},
+	{
+		Name: "traefik",
+		URL:  "https://helm.traefik.io/traefik",
+	},
 }
 
 var ServerLabels = map[string]string{

plugins/scheduler-k3s/subcommands.go

@@ -151,6 +151,8 @@ func CommandInitialize(serverIP string, taintScheduling bool) error {
 		"--cluster-init",
 		// disable local-storage
 		"--disable", "local-storage",
+		// disable traefik so it can be installed separately
+		"--disable", "traefik",
 		// expose etcd metrics
 		"--etcd-expose-metrics",
 		// use wireguard for flannel
@@ -239,17 +241,6 @@ func CommandInitialize(serverIP string, taintScheduling bool) error {
 		}
 	}
 
-	common.LogInfo2Quiet("Updating traefik config")
-	contents, err := templates.ReadFile("templates/helm-config/traefik-config.yaml")
-	if err != nil {
-		return fmt.Errorf("Unable to read traefik config template: %w", err)
-	}
-
-	err = os.WriteFile("/var/lib/rancher/k3s/server/manifests/traefik-custom.yaml", contents, 0600)
-	if err != nil {
-		return fmt.Errorf("Unable to write traefik config: %w", err)
-	}
-
 	common.LogInfo2Quiet("Installing helm charts")
 	err = installHelmCharts(ctx, clientset)
 	if err != nil {

plugins/scheduler-k3s/templates/helm-config/traefik-config.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: traefik-crd
-  namespace: kube-system
-spec:
-  bootstrap: true
----
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: traefik
-  namespace: kube-system
-spec:
-  bootstrap: true

plugins/scheduler-k3s/templates/helm-config/traefik.yaml

@@ -0,0 +1,26 @@
+logs:
+  access:
+    enabled: true
+    format: json
+podAnnotations:
+  prometheus.io/port: "8082"
+  prometheus.io/scrape: "true"
+providers:
+  kubernetesCRD:
+    enabled: true
+    allowCrossNamespace: true
+  kubernetesIngress:
+    publishedService:
+      enabled: true
+priorityClassName: "system-cluster-critical"
+tolerations:
+- key: "CriticalAddonsOnly"
+  operator: "Exists"
+- key: "node-role.kubernetes.io/control-plane"
+  operator: "Exists"
+  effect: "NoSchedule"
+- key: "node-role.kubernetes.io/master"
+  operator: "Exists"
+  effect: "NoSchedule"
+service:
+  ipFamilyPolicy: "PreferDualStack"