fix: ensure critical addons come up when the node is tainted

Rather than taint on `node-role.kubernetes.io/master=true:NoSchedule`, we check for `CriticalAddonsOnly=true:NoSchedule` and taint on that.

plugins/scheduler-k3s/functions.go

@@ -3,6 +3,7 @@ package scheduler_k3s
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"sort"
@@ -11,6 +12,7 @@ import (
 	"time"
 
 	"github.com/dokku/dokku/plugins/common"
+	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -116,17 +118,6 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 		return err
 	}
 
-	namespaces, err := clientset.ListNamespaces(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, namespace := range namespaces {
-		if namespace.Name == namespaceName {
-			return nil
-		}
-	}
-
 	namespace := corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: namespaceName,
@@ -139,7 +130,7 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 		},
 	}
 	_, err = clientset.CreateNamespace(ctx, CreateNamespaceInput{
-		Namespace: namespace,
+		Name: namespace,
 	})
 	if err != nil {
 		return err
@@ -322,6 +313,73 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
+func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
+	for _, repo := range HelmRepositories {
+		helmAgent, err := NewHelmAgent("default", DeployLogPrinter)
+		if err != nil {
+			return fmt.Errorf("Error creating helm agent: %w", err)
+		}
+
+		err = helmAgent.AddRepository(ctx, AddRepositoryInput(repo))
+		if err != nil {
+			return fmt.Errorf("Error adding helm repository %s: %w", repo.Name, err)
+		}
+	}
+
+	for _, chart := range HelmCharts {
+		if chart.CreateNamespace {
+			namespace := corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: chart.Namespace,
+					Annotations: map[string]string{
+						"dokku.com/managed": "true",
+					},
+					Labels: map[string]string{
+						"dokku.com/managed": "true",
+					},
+				},
+			}
+			_, err := clientset.CreateNamespace(ctx, CreateNamespaceInput{
+				Name: namespace,
+			})
+			if err != nil {
+				return fmt.Errorf("Error creating namespace %s: %w", chart.Namespace, err)
+			}
+		}
+
+		contents, err := templates.ReadFile(fmt.Sprintf("templates/%s.yaml", chart.ReleaseName))
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err)
+		}
+
+		var values map[string]interface{}
+		if len(contents) > 0 {
+			err = yaml.Unmarshal(contents, &values)
+			if err != nil {
+				return fmt.Errorf("Error unmarshalling values file: %w", err)
+			}
+		}
+
+		helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter)
+		if err != nil {
+			return fmt.Errorf("Error creating helm agent: %w", err)
+		}
+
+		err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
+			ChartPath:   chart.ChartPath,
+			Namespace:   chart.Namespace,
+			ReleaseName: chart.ReleaseName,
+			RepoURL:     chart.RepoURL,
+			Values:      values,
+			Version:     chart.Version,
+		})
+		if err != nil {
+			return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err)
+		}
+	}
+	return nil
+}
+
 func isK3sInstalled() error {
 	if !common.FileExists("/usr/local/bin/k3s") {
 		return fmt.Errorf("k3s binary is not available")
@@ -340,7 +398,7 @@ func isK3sInstalled() error {
 
 func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
 	return func(ctx context.Context) (bool, error) {
-		fmt.Printf(".") // progress bar!
+		fmt.Printf(".")
 
 		pod, err := clientset.GetPod(ctx, GetPodInput{
 			Name:      podName,

plugins/scheduler-k3s/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/fatih/color v1.16.0
 	github.com/go-openapi/jsonpointer v0.19.6
 	github.com/go-resty/resty/v2 v2.11.0
+	github.com/gofrs/flock v0.8.1
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/rancher/wharfie v0.6.5

plugins/scheduler-k3s/go.sum

@@ -155,6 +155,8 @@ github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XE
 github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=

plugins/scheduler-k3s/helm.go

@@ -4,18 +4,23 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"os"
-	"os/signal"
+	"path/filepath"
 	"sort"
 	"strings"
-	"syscall"
 	"time"
 	"unicode"
 
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/gofrs/flock"
+	"gopkg.in/yaml.v3"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/kube"
+	"helm.sh/helm/v3/pkg/repo"
 	"helm.sh/helm/v3/pkg/storage/driver"
 )
 
@@ -43,8 +48,10 @@ type ChartInput struct {
 	ChartPath         string
 	Namespace         string
 	ReleaseName       string
+	RepoURL           string
 	RollbackOnFailure bool
 	Timeout           time.Duration
+	Version           string
 	Values            map[string]interface{}
 }
 
@@ -80,7 +87,74 @@ func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error)
 	}, nil
 }
 
+type AddRepositoryInput struct {
+	Name string
+	URL  string
+}
+
+func (h *HelmAgent) AddRepository(ctx context.Context, helmRepo AddRepositoryInput) error {
+	settings := cli.New()
+	repoFile := settings.RepositoryConfig
+
+	err := os.MkdirAll(filepath.Dir(repoFile), os.ModePerm)
+	if err != nil && !os.IsExist(err) {
+		return fmt.Errorf("Error creating repository directory: %w", err)
+	}
+
+	fileLock := flock.New(strings.Replace(repoFile, filepath.Ext(repoFile), ".lock", 1))
+	locked, err := fileLock.TryLockContext(ctx, time.Second)
+	if err != nil {
+		return fmt.Errorf("Error acquiring file lock: %w", err)
+	}
+
+	if !locked {
+		return fmt.Errorf("Could not acquire file lock")
+	}
+
+	defer fileLock.Unlock() // nolint: errcheck
+
+	b, err := os.ReadFile(repoFile)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("Error reading repository file: %w", err)
+	}
+
+	var f repo.File
+	if err := yaml.Unmarshal(b, &f); err != nil {
+		return fmt.Errorf("Error unmarshalling yaml: %w", err)
+	}
+
+	if f.Has(helmRepo.Name) {
+		return nil
+	}
+
+	c := repo.Entry{
+		Name: helmRepo.Name,
+		URL:  helmRepo.URL,
+	}
+
+	r, err := repo.NewChartRepository(&c, getter.All(settings))
+	if err != nil {
+		return fmt.Errorf("Error creating chart repository: %w", err)
+	}
+
+	if _, err := r.DownloadIndexFile(); err != nil {
+		return fmt.Errorf("Specified repository '%q' is not a valid chart repository or cannot be reached: %w", helmRepo.URL, err)
+	}
+
+	f.Update(&c)
+
+	if err := f.WriteFile(repoFile, 0644); err != nil {
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
 func (h *HelmAgent) ChartExists(releaseName string) (bool, error) {
+	if releaseName == "" {
+		return false, fmt.Errorf("Release name is required")
+	}
+
 	client := action.NewHistory(h.Configuration)
 	client.Max = 1
 	releases, err := client.Run(releaseName)
@@ -98,14 +172,14 @@ func (h *HelmAgent) ChartExists(releaseName string) (bool, error) {
 	return false, nil
 }
 
-func (h *HelmAgent) DeleteRevision(releaseName string, revision int) error {
+func (h *HelmAgent) DeleteRevision(ctx context.Context, releaseName string, revision int) error {
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
 	secretName := fmt.Sprintf("sh.helm.release.v1.%s.v%d", releaseName, revision)
-	err = clientset.DeleteSecret(context.Background(), DeleteSecretInput{
+	err = clientset.DeleteSecret(ctx, DeleteSecretInput{
 		Name:      secretName,
 		Namespace: h.Namespace,
 	})
@@ -126,7 +200,7 @@ func (h *HelmAgent) GetValues(releaseName string) (map[string]interface{}, error
 	return values, nil
 }
 
-func (h *HelmAgent) InstallOrUpgradeChart(input ChartInput) error {
+func (h *HelmAgent) InstallOrUpgradeChart(ctx context.Context, input ChartInput) error {
 	chartExists, err := h.ChartExists(input.ReleaseName)
 	if err != nil {
 		return fmt.Errorf("Error checking if chart exists: %w", err)
@@ -134,14 +208,14 @@ func (h *HelmAgent) InstallOrUpgradeChart(input ChartInput) error {
 
 	if chartExists {
 		common.LogExclaim(fmt.Sprintf("Upgrading %s", input.ReleaseName))
-		return h.UpgradeChart(input)
+		return h.UpgradeChart(ctx, input)
 	}
 
 	common.LogExclaim(fmt.Sprintf("Installing %s", input.ReleaseName))
-	return h.InstallChart(input)
+	return h.InstallChart(ctx, input)
 }
 
-func (h *HelmAgent) InstallChart(input ChartInput) error {
+func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error {
 	namespace := input.Namespace
 	if namespace == "" {
 		namespace = h.Namespace
@@ -167,7 +241,15 @@ func (h *HelmAgent) InstallChart(input ChartInput) error {
 	client.Timeout = input.Timeout
 	client.Wait = false
 
-	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil)
+	settings := cli.New()
+	if input.RepoURL != "" {
+		client.ChartPathOptions.RepoURL = input.RepoURL
+	}
+	if input.Version != "" {
+		client.ChartPathOptions.Version = input.Version
+	}
+
+	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, settings)
 	if err != nil {
 		return fmt.Errorf("Error locating chart: %w", err)
 	}
@@ -177,16 +259,6 @@ func (h *HelmAgent) InstallChart(input ChartInput) error {
 		return fmt.Errorf("Error loading chart: %w", err)
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	cSignal := make(chan os.Signal, 2)
-	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-cSignal
-		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", input.ReleaseName))
-		cancel()
-	}()
-
 	_, err = client.RunWithContext(ctx, chartRequested, input.Values)
 	if err != nil {
 		return fmt.Errorf("Error deploying: %w", err)
@@ -195,7 +267,7 @@ func (h *HelmAgent) InstallChart(input ChartInput) error {
 	return nil
 }
 
-func (h *HelmAgent) ListRevisions(releaseName string) ([]Release, error) {
+func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Release, error) {
 	client := action.NewHistory(h.Configuration)
 	response, err := client.Run(releaseName)
 	if err != nil {
@@ -218,7 +290,7 @@ func (h *HelmAgent) ListRevisions(releaseName string) ([]Release, error) {
 	return releases, nil
 }
 
-func (h *HelmAgent) UpgradeChart(input ChartInput) error {
+func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
 	namespace := input.Namespace
 	if namespace == "" {
 		namespace = h.Namespace
@@ -243,6 +315,9 @@ func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 	client.Namespace = namespace
 	client.Timeout = input.Timeout
 	client.Wait = false
+	if input.RepoURL != "" {
+		client.RepoURL = input.RepoURL
+	}
 
 	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil)
 	if err != nil {
@@ -254,16 +329,6 @@ func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 		return fmt.Errorf("Error loading chart: %w", err)
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	cSignal := make(chan os.Signal, 2)
-	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-cSignal
-		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", input.ReleaseName))
-		cancel()
-	}()
-
 	_, err = client.RunWithContext(ctx, input.ReleaseName, chartRequested, input.Values)
 	if err != nil {
 		return fmt.Errorf("Error deploying: %w", err)

plugins/scheduler-k3s/k8s.go

@@ -154,13 +154,24 @@ func (k KubernetesClient) CreateJob(ctx context.Context, input CreateJobInput) (
 
 // CreateNamespaceInput contains all the information needed to create a Kubernetes namespace
 type CreateNamespaceInput struct {
-	// Namespace is the Kubernetes namespace
-	Namespace v1.Namespace
+	// Name is the name of the Kubernetes namespace
+	Name v1.Namespace
 }
 
 // CreateNamespace creates a Kubernetes namespace
 func (k KubernetesClient) CreateNamespace(ctx context.Context, input CreateNamespaceInput) (v1.Namespace, error) {
-	namespace, err := k.Client.CoreV1().Namespaces().Create(ctx, &input.Namespace, metav1.CreateOptions{})
+	namespaces, err := k.ListNamespaces(ctx)
+	if err != nil {
+		return v1.Namespace{}, err
+	}
+
+	for _, namespace := range namespaces {
+		if namespace.Name == input.Name.Name {
+			return namespace, nil
+		}
+	}
+
+	namespace, err := k.Client.CoreV1().Namespaces().Create(ctx, &input.Name, metav1.CreateOptions{})
 	if err != nil {
 		return v1.Namespace{}, err
 	}

plugins/scheduler-k3s/scheduler_k3s.go

@@ -1,6 +1,7 @@
 package scheduler_k3s
 
 import (
+	"embed"
 	"sync"
 
 	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
@@ -49,24 +50,70 @@ type Manifest struct {
 	Path    string
 }
 
-var Manifests = []Manifest{
-	{
-		Name:    "longhorn",
-		Version: "1.5.3",
-		Path:    "https://raw.githubusercontent.com/longhorn/longhorn/v1.5.3/deploy/longhorn.yaml",
-	},
+var KubernetesManifests = []Manifest{
 	{
 		Name:    "system-upgrader",
 		Version: "0.13.2",
-		Path:    "https://github.com/rancher/system-upgrade-controller/releases/v0.13.2/download/system-upgrade-controller.yaml",
-	},
-	{
-		Name:    "cert-manager",
-		Version: "v1.13.3",
-		Path:    "https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml",
+		Path:    "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.2/system-upgrade-controller.yaml",
 	},
 }
 
+type HelmChart struct {
+	ReleaseName     string
+	ChartPath       string
+	Namespace       string
+	CreateNamespace bool
+	Version         string
+	Path            string
+	RepoURL         string
+}
+
+var HelmCharts = []HelmChart{
+	{
+		ChartPath:       "cert-manager",
+		CreateNamespace: true,
+		Namespace:       "cert-manager",
+		ReleaseName:     "cert-manager",
+		RepoURL:         "https://charts.jetstack.io",
+		Version:         "v1.13.3",
+	},
+	{
+		ChartPath:       "longhorn",
+		CreateNamespace: true,
+		Namespace:       "longhorn-system",
+		ReleaseName:     "longhorn",
+		RepoURL:         "https://charts.longhorn.io",
+		Version:         "1.5.3",
+	},
+}
+
+type HelmRepository struct {
+	Name string
+	URL  string
+}
+
+var HelmRepositories = []HelmRepository{
+	{
+		Name: "jetstack",
+		URL:  "https://charts.jetstack.io",
+	},
+	{
+		Name: "longhorn",
+		URL:  "https://charts.longhorn.io",
+	},
+}
+
+var ServerLabels = map[string]string{
+	"svccontroller.k3s.cattle.io/enablelb": "true",
+}
+
+var WorkerLabels = map[string]string{
+	"node-role.kubernetes.io/role": "worker",
+}
+
+//go:embed templates/*
+var templates embed.FS
+
 func init() {
 	k8sNativeSchemeOnce.Do(func() {
 		_ = appsv1.AddToScheme(runtimeScheme)

plugins/scheduler-k3s/subcommands.go

@@ -8,7 +8,9 @@ import (
 	"net"
 	"net/url"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 
 	"github.com/dokku/dokku/plugins/common"
 	resty "github.com/go-resty/resty/v2"
@@ -21,6 +23,17 @@ func CommandInitialize(taintScheduling bool) error {
 		return fmt.Errorf("k3s already installed, cannot re-initialize k3s")
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
 	networkInterface := common.PropertyGetDefault("scheduler-k3s", "--global", "network-interface", "eth0")
 	ifaces, err := net.Interfaces()
 	if err != nil {
@@ -176,7 +189,7 @@ func CommandInitialize(taintScheduling bool) error {
 		"--token", token,
 	}
 	if taintScheduling {
-		args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
+		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
 	}
 
 	common.LogInfo2Quiet("Running k3s installer")
@@ -214,9 +227,13 @@ func CommandInitialize(taintScheduling bool) error {
 	}
 
 	clientset, err := NewKubernetesClient()
-	for _, manifest := range Manifests {
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	for _, manifest := range KubernetesManifests {
 		common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version))
-		err = clientset.ApplyKubernetesManifest(context.Background(), ApplyKubernetesManifestInput{
+		err = clientset.ApplyKubernetesManifest(ctx, ApplyKubernetesManifestInput{
 			Manifest: manifest.Path,
 		})
 		if err != nil {
@@ -224,18 +241,37 @@ func CommandInitialize(taintScheduling bool) error {
 		}
 	}
 
-	common.LogInfo2Quiet("Labeling node svccontroller.k3s.cattle.io/enablelb=true")
-	if err != nil {
-		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	for key, value := range ServerLabels {
+		common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))
+		if err != nil {
+			return fmt.Errorf("Unable to create kubernetes client: %w", err)
+		}
+
+		err = clientset.LabelNode(context.Background(), LabelNodeInput{
+			Name:  nodeName,
+			Key:   key,
+			Value: value,
+		})
+		if err != nil {
+			return fmt.Errorf("Unable to patch node: %w", err)
+		}
 	}
 
-	err = clientset.LabelNode(context.Background(), LabelNodeInput{
-		Name:  nodeName,
-		Key:   "svccontroller.k3s.cattle.io/enablelb",
-		Value: "true",
-	})
+	common.LogInfo2Quiet("Updating traefik config")
+	contents, err := templates.ReadFile("templates/traefik-config.yaml")
 	if err != nil {
-		return fmt.Errorf("Unable to patch node: %w", err)
+		return fmt.Errorf("Unable to read traefik config template: %w", err)
+	}
+
+	err = os.WriteFile("/var/lib/rancher/k3s/server/manifests/traefik-custom.yaml", contents, 0600)
+	if err != nil {
+		return fmt.Errorf("Unable to write traefik config: %w", err)
+	}
+
+	common.LogInfo2Quiet("Installing helm charts")
+	err = installHelmCharts(ctx, clientset)
+	if err != nil {
+		return fmt.Errorf("Unable to install helm charts: %w", err)
 	}
 
 	common.LogVerboseQuiet("Done")
@@ -449,8 +485,9 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		// bind proxy metrics to all interfaces
 		args = append(args, "--kube-proxy-arg", "metrics-bind-address=0.0.0.0")
 	}
+
 	if taintScheduling {
-		args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
+		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
 	}
 
 	common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
@@ -492,22 +529,21 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
 	}
 
+	labels := ServerLabels
 	if role == "worker" {
-		common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker")
-		err = clientset.LabelNode(ctx, LabelNodeInput{
-			Name:  nodes[0].Name,
-			Key:   "kubernetes.io/role",
-			Value: "worker",
-		})
+		labels = WorkerLabels
+	}
+
+	for key, value := range labels {
+		common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))
 		if err != nil {
-			return fmt.Errorf("Unable to patch node: %w", err)
+			return fmt.Errorf("Unable to create kubernetes client: %w", err)
 		}
-	} else {
-		common.LogInfo2Quiet("Labeling node svccontroller.k3s.cattle.io/enablelb=true")
-		err = clientset.LabelNode(ctx, LabelNodeInput{
-			Name:  nodes[0].Name,
-			Key:   "svccontroller.k3s.cattle.io/enablelb",
-			Value: "true",
+
+		err = clientset.LabelNode(context.Background(), LabelNodeInput{
+			Name:  nodeName,
+			Key:   key,
+			Value: value,
 		})
 		if err != nil {
 			return fmt.Errorf("Unable to patch node: %w", err)

plugins/scheduler-k3s/templates/cert-manager.yaml

@@ -0,0 +1,28 @@
+installCRDs: true
+
+tolerations:
+- key: CriticalAddonsOnly
+  operator: Equal
+  value: "true"
+  effect: "NoSchedule"
+
+webhook:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+cainjector:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+startupapicheck:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"

plugins/scheduler-k3s/templates/longhorn.yaml

@@ -0,0 +1,41 @@
+defaultSettings:
+  taintToleration: "CriticalAddonsOnly=true:NoSchedule"
+
+longhornManager:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornDriver:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornUI:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornConversionWebhook:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornAdmissionWebhook:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornRecoveryBackend:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"

plugins/scheduler-k3s/templates/traefik-config.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: traefik-crd
+  namespace: kube-system
+spec:
+  bootstrap: true
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: traefik
+  namespace: kube-system
+spec:
+  bootstrap: true

plugins/scheduler-k3s/triggers.go

@@ -167,6 +167,15 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	}
 
 	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	cSignal := make(chan os.Signal, 2)
+	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-cSignal
+		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", appName))
+		cancel()
+	}()
+
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 	if err := createKubernetesNamespace(ctx, namespace); err != nil {
 		return fmt.Errorf("Error creating kubernetes namespace for deployment: %w", err)
@@ -455,7 +464,7 @@ data:
 		return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
 	}
 
-	err = helmAgent.InstallOrUpgradeChart(ChartInput{
+	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
 		ChartPath:         chartPath,
 		Namespace:         namespace,
 		ReleaseName:       fmt.Sprintf("dokku-%s", appName),