feat: add scheduler-k3s:ensure-charts

This command can be used to ensure charts are properly installed and configured according to Dokku's internal settings.

docs/deployment/schedulers/k3s.md

@@ -10,6 +10,7 @@ scheduler-k3s:autoscaling-auth:report <app|--global> [--format stdout|json] [--i
 scheduler-k3s:cluster-add [ssh://user@host:port]    # Adds a server node to a Dokku-managed cluster
 scheduler-k3s:cluster-list                          # Lists all nodes in a Dokku-managed cluster
 scheduler-k3s:cluster-remove [node-id]              # Removes client node to a Dokku-managed cluster
+scheduler-k3s:ensure-charts                         # Ensures the k3s charts are installed
 scheduler-k3s:initialize                            # Initializes a cluster
 scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
 scheduler-k3s:report [<app>] [<flag>]               # Displays a scheduler-k3s report for one or more apps

plugins/scheduler-k3s/Makefile

@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
+SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
 TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s

plugins/scheduler-k3s/helm.go

@@ -21,6 +21,7 @@ import (
 	"helm.sh/helm/v3/pkg/cli"
 	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/kube"
+	"helm.sh/helm/v3/pkg/release"
 	"helm.sh/helm/v3/pkg/repo"
 	"helm.sh/helm/v3/pkg/storage/driver"
 )
@@ -58,9 +59,12 @@ type ChartInput struct {
 }
 
 type Release struct {
-	Name      string
-	Namespace string
-	Version   int
+	AppVersion string
+	Name       string
+	Namespace  string
+	Revision   int
+	Status     release.Status
+	Version    string
 }
 
 type HelmAgent struct {
@@ -269,24 +273,61 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error {
 	return nil
 }
 
-func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Release, error) {
-	client := action.NewHistory(h.Configuration)
-	response, err := client.Run(releaseName)
+func (h *HelmAgent) InstalledRevision(releaseName string) (Release, error) {
+	revisions, err := h.ListRevisions(ListRevisionsInput{
+		ReleaseName: releaseName,
+		Max:         1,
+	})
 	if err != nil {
-		return nil, fmt.Errorf("Error getting revisions: %w", err)
+		return Release{}, err
+	}
+
+	if len(revisions) == 0 {
+		return Release{}, nil
+	}
+
+	return revisions[0], nil
+}
+
+type ListRevisionsInput struct {
+	ReleaseName string
+	Max         int
+}
+
+func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) {
+	client := action.NewHistory(h.Configuration)
+	if input.Max > 0 {
+		client.Max = input.Max
 	}
 
 	releases := []Release{}
+	response, err := client.Run(input.ReleaseName)
+	if err != nil {
+		if errors.Is(err, driver.ErrReleaseNotFound) {
+			return releases, nil
+		}
+
+		return nil, fmt.Errorf("Error getting revisions: %w", err)
+	}
+
 	for _, release := range response {
+		appVersion := "MISSING"
+		if release.Chart != nil && release.Chart.Metadata != nil {
+			appVersion = release.Chart.AppVersion()
+		}
+
 		releases = append(releases, Release{
-			Name:      release.Name,
-			Namespace: release.Namespace,
-			Version:   release.Version,
+			AppVersion: appVersion,
+			Name:       release.Name,
+			Namespace:  release.Namespace,
+			Revision:   release.Version,
+			Status:     release.Info.Status,
+			Version:    release.Chart.Metadata.Version,
 		})
 	}
 
 	sort.Slice(releases, func(i, j int) bool {
-		return releases[i].Version < releases[j].Version
+		return releases[i].Revision < releases[j].Revision
 	})
 
 	return releases, nil

plugins/scheduler-k3s/src/commands/commands.go

@@ -22,6 +22,7 @@ Additional commands:`
     scheduler-k3s:cluster-add [--insecure-allow-unknown-hosts] [--server-ip SERVER_IP] [--taint-scheduling] <ssh://user@host:port>, Adds a server node to a Dokku-managed cluster
     scheduler-k3s:cluster-list [--format json|stdout], Lists all nodes in a Dokku-managed cluster
     scheduler-k3s:cluster-remove [node-id], Removes client node to a Dokku-managed cluster
+	scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
     scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster
     scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
     scheduler-k3s:report [<app>] [<flag>], Displays a scheduler-k3s report for one or more apps

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -73,6 +73,10 @@ func main() {
 		args.Parse(os.Args[2:])
 		nodeName := args.Arg(0)
 		err = scheduler_k3s.CommandClusterRemove(nodeName)
+	case "ensure-charts":
+		args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		err = scheduler_k3s.CommandEnsureCharts()
 	case "initialize":
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")

plugins/scheduler-k3s/subcommands.go

@@ -798,6 +798,96 @@ func CommandClusterRemove(nodeName string) error {
 	return nil
 }
 
+// CommandEnsureCharts ensures that the required helm charts are installed
+func CommandEnsureCharts() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	ingressClass := getGlobalIngressClass()
+
+	namespacedHelmAgents := map[string]*HelmAgent{}
+	for _, chart := range HelmCharts {
+		_, ok := namespacedHelmAgents[chart.Namespace]
+		if !ok {
+			helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter)
+			if err != nil {
+				common.LogWarn(fmt.Sprintf("Unable to create helm agent: %s", err.Error()))
+				return err
+			}
+			namespacedHelmAgents[chart.Namespace] = helmAgent
+		}
+	}
+
+	common.LogInfo2Quiet("Installing helm charts")
+	err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool {
+		common.LogInfo1(fmt.Sprintf("Processing chart %s@%s", chart.ReleaseName, chart.Version))
+		if chart.ChartPath == "traefik" && ingressClass == "nginx" {
+			common.LogVerbose("Skipping chart due to ingress-class mismatch")
+			return false
+		}
+
+		if chart.ChartPath == "ingress-nginx" && ingressClass == "traefik" {
+			common.LogVerbose("Skipping chart due to ingress-class mismatch")
+			return false
+		}
+
+		if chart.ReleaseName == "keda-add-ons-http" {
+			common.LogVerbose("Forcing install of keda-add-ons-http")
+			return true
+		}
+
+		helmAgent := namespacedHelmAgents[chart.Namespace]
+		latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName)
+		if err != nil {
+			common.LogWarn(fmt.Sprintf("Unable to get installed revision: %s", err))
+			return false
+		}
+
+		if latestRevision.Name == "" {
+			common.LogVerbose("Installing missing chart")
+			return true
+		}
+
+		if latestRevision.Version != chart.Version {
+			common.LogVerbose(fmt.Sprintf("Installing chart due to version mismatch: %s != %s", latestRevision.AppVersion, chart.Version))
+			return false
+		}
+
+		common.LogVerbose("Skipping chart: already installed")
+		return false
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to install helm charts: %w", err)
+	}
+
+	for _, manifest := range KubernetesManifests {
+		common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version))
+		err = clientset.ApplyKubernetesManifest(ctx, ApplyKubernetesManifestInput{
+			Manifest: manifest.Path,
+		})
+		if err != nil {
+			return fmt.Errorf("Unable to apply kubernetes manifest: %w", err)
+		}
+	}
+
+	common.LogInfo2Quiet("Done")
+
+	return nil
+}
+
 // CommandLabelsSet set or clear a scheduler-k3s label for an app
 func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error {
 	if resourceType == "" {