feat: add version-aware helm chart upgrade callbacks

Some helm chart upgrades require side-effects that a plain `helm upgrade` cannot perform, such as deleting deployments whose immutable selectors changed. Charts may now register pre and post upgrade hooks against a target version; applicable hooks fire in ascending semver order, each bracketed around an upgrade to its version, with a final upgrade to the chart's configured version when needed. The new `keda-add-ons-http` 0.12.2 hook deletes the chart-managed deployments before the upgrade so the new selectors take effect cleanly.

docs/deployment/schedulers/k3s.md

@@ -683,6 +683,14 @@ Alternatively, a comma separated list of chart names can be specified to only fo
 dokku scheduler-k3s:ensure-charts --charts cert-manager
 ```
 
+### Chart upgrade callbacks
+
+Some chart version transitions require side-effects that a plain `helm upgrade` cannot perform. For example, the upgrade of `keda-add-ons-http` to `0.12.2` introduces breaking changes to deployment selectors, which are immutable in Kubernetes; the chart-managed deployments must be deleted before the upgrade can proceed.
+
+Dokku handles these cases internally by registering version-targeted pre-upgrade and post-upgrade callbacks against affected charts. When `scheduler-k3s:ensure-charts` runs, the installed chart version is compared against the registered callbacks: any whose target version is greater than the currently installed version and at most the configured chart version are executed in ascending semver order, each bracketed around an upgrade to that intermediate version. A final upgrade to the configured chart version then completes the run.
+
+These callbacks are not user-configurable. Failures during a callback or its surrounding upgrade abort the run, and a subsequent `scheduler-k3s:ensure-charts` resumes from the current state.
+
 ## Scheduler Interface
 
 The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler.

plugins/scheduler-k3s/chart_hooks.go

@@ -0,0 +1,104 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/Masterminds/semver/v3"
+)
+
+// ChartHookFunc is the signature for a function that runs around a helm chart
+// upgrade. It receives the active Kubernetes client, the chart definition, and
+// the helm release that was installed before the upgrade started.
+type ChartHookFunc func(ctx context.Context, clientset KubernetesClient, chart HelmChart, installedRevision Release) error
+
+// ChartHook attaches version-specific behaviour to a helm chart upgrade. A
+// hook fires when the currently installed release version is below
+// TargetVersion and the chart's configured target version is at or above
+// TargetVersion. PreUpgrade runs before the chart is upgraded to
+// TargetVersion; PostUpgrade runs after that intermediate upgrade succeeds.
+type ChartHook struct {
+	TargetVersion string
+	PreUpgrade    ChartHookFunc
+	PostUpgrade   ChartHookFunc
+}
+
+// ChartHooks is keyed by HelmChart.ReleaseName. selectChartHooks resorts
+// entries by semver, so authoring order is informative rather than load-bearing.
+var ChartHooks = map[string][]ChartHook{
+	"keda-add-ons-http": {
+		{
+			TargetVersion: "0.12.2",
+			PreUpgrade:    deleteKedaAddOnsHttpDeployments,
+		},
+	},
+}
+
+// selectChartHooks returns the hooks for releaseName whose TargetVersion is
+// strictly greater than installedVersion and less than or equal to
+// targetVersion, sorted ascending by TargetVersion. An empty installedVersion
+// signals a fresh install and yields no hooks.
+func selectChartHooks(releaseName, installedVersion, targetVersion string) ([]ChartHook, error) {
+	if installedVersion == "" {
+		return nil, nil
+	}
+
+	hooks, ok := ChartHooks[releaseName]
+	if !ok || len(hooks) == 0 {
+		return nil, nil
+	}
+
+	installed, err := semver.NewVersion(installedVersion)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid installed version %q for chart %s: %w", installedVersion, releaseName, err)
+	}
+
+	target, err := semver.NewVersion(targetVersion)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid target version %q for chart %s: %w", targetVersion, releaseName, err)
+	}
+
+	type indexedHook struct {
+		version *semver.Version
+		hook    ChartHook
+	}
+
+	selected := []indexedHook{}
+	for _, hook := range hooks {
+		hookVersion, err := semver.NewVersion(hook.TargetVersion)
+		if err != nil {
+			return nil, fmt.Errorf("Invalid hook TargetVersion %q for chart %s: %w", hook.TargetVersion, releaseName, err)
+		}
+
+		if hookVersion.Compare(installed) <= 0 {
+			continue
+		}
+		if hookVersion.Compare(target) > 0 {
+			continue
+		}
+
+		selected = append(selected, indexedHook{version: hookVersion, hook: hook})
+	}
+
+	sort.Slice(selected, func(i, j int) bool {
+		return selected[i].version.LessThan(selected[j].version)
+	})
+
+	out := make([]ChartHook, 0, len(selected))
+	for _, entry := range selected {
+		out = append(out, entry.hook)
+	}
+	return out, nil
+}
+
+// deleteKedaAddOnsHttpDeployments removes the deployments managed by the
+// keda-add-ons-http helm release so the chart can recreate them with the
+// updated selectors introduced in 0.12.x. Selectors on Deployments are
+// immutable, so a straight helm upgrade fails without this step.
+func deleteKedaAddOnsHttpDeployments(ctx context.Context, clientset KubernetesClient, chart HelmChart, installedRevision Release) error {
+	return clientset.DeleteDeployment(ctx, DeleteDeploymentInput{
+		Namespace:     chart.Namespace,
+		LabelSelector: "app.kubernetes.io/instance=keda-add-ons-http",
+	})
+}

plugins/scheduler-k3s/chart_hooks_test.go

@@ -0,0 +1,159 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"testing"
+)
+
+func makeHook(version string) ChartHook {
+	noop := func(context.Context, KubernetesClient, HelmChart, Release) error { return nil }
+	return ChartHook{
+		TargetVersion: version,
+		PreUpgrade:    noop,
+	}
+}
+
+func versionsOf(hooks []ChartHook) []string {
+	out := make([]string, 0, len(hooks))
+	for _, hook := range hooks {
+		out = append(out, hook.TargetVersion)
+	}
+	return out
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func TestSelectChartHooks(t *testing.T) {
+	previous := ChartHooks
+	t.Cleanup(func() { ChartHooks = previous })
+
+	ChartHooks = map[string][]ChartHook{
+		"chart-with-hooks": {
+			makeHook("0.12.0"),
+			makeHook("0.10.0"),
+			makeHook("0.11.0"),
+		},
+	}
+
+	cases := []struct {
+		name             string
+		releaseName      string
+		installedVersion string
+		targetVersion    string
+		want             []string
+		wantErr          bool
+	}{
+		{
+			name:             "fresh install yields no hooks",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "",
+			targetVersion:    "0.12.0",
+			want:             nil,
+		},
+		{
+			name:             "no hooks registered for release",
+			releaseName:      "chart-without-hooks",
+			installedVersion: "0.5.0",
+			targetVersion:    "0.6.0",
+			want:             nil,
+		},
+		{
+			name:             "single applicable hook returned",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.10.0",
+			targetVersion:    "0.11.0",
+			want:             []string{"0.11.0"},
+		},
+		{
+			name:             "all hooks below installed version excluded",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.12.0",
+			targetVersion:    "0.12.0",
+			want:             nil,
+		},
+		{
+			name:             "all applicable hooks returned in ascending semver order",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.9.0",
+			targetVersion:    "0.12.0",
+			want:             []string{"0.10.0", "0.11.0", "0.12.0"},
+		},
+		{
+			name:             "partial coverage returns only unsatisfied hooks",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.10.0",
+			targetVersion:    "0.12.0",
+			want:             []string{"0.11.0", "0.12.0"},
+		},
+		{
+			name:             "target above all hooks still caps at chart target",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.9.0",
+			targetVersion:    "0.11.0",
+			want:             []string{"0.10.0", "0.11.0"},
+		},
+		{
+			name:             "unparseable installed version errors",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "not-a-version",
+			targetVersion:    "0.12.0",
+			wantErr:          true,
+		},
+		{
+			name:             "unparseable target version errors",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.9.0",
+			targetVersion:    "not-a-version",
+			wantErr:          true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := selectChartHooks(tc.releaseName, tc.installedVersion, tc.targetVersion)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("expected error, got nil")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			gotVersions := versionsOf(got)
+			if !equalStrings(gotVersions, tc.want) {
+				t.Fatalf("versions mismatch: got %v, want %v", gotVersions, tc.want)
+			}
+		})
+	}
+}
+
+func TestSelectChartHooksTreatsLeadingVAsEqual(t *testing.T) {
+	previous := ChartHooks
+	t.Cleanup(func() { ChartHooks = previous })
+
+	ChartHooks = map[string][]ChartHook{
+		"chart-v-prefixed": {
+			makeHook("v1.13.3"),
+		},
+	}
+
+	got, err := selectChartHooks("chart-v-prefixed", "v1.12.0", "v1.13.3")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(got) != 1 || got[0].TargetVersion != "v1.13.3" {
+		t.Fatalf("expected v1.13.3 hook, got %v", versionsOf(got))
+	}
+}

plugins/scheduler-k3s/functions.go

@@ -1679,17 +1679,57 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 			return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
 		}
 
-		err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
-			ChartPath:   chart.ChartPath,
-			Namespace:   chart.Namespace,
-			ReleaseName: chart.ReleaseName,
-			RepoURL:     chart.RepoURL,
-			Values:      values,
-			Version:     chart.Version,
-			Timeout:     timeoutDuration,
-			Wait:        true,
-		})
+		installedRevision, err := helmAgent.InstalledRevision(chart.ReleaseName)
 		if err != nil {
+			return fmt.Errorf("Error getting installed revision for %s: %w", chart.ReleaseName, err)
+		}
+
+		hooks, err := selectChartHooks(chart.ReleaseName, installedRevision.Version, chart.Version)
+		if err != nil {
+			return fmt.Errorf("Error selecting upgrade hooks for chart %s: %w", chart.ReleaseName, err)
+		}
+
+		buildChartInput := func(version string) ChartInput {
+			return ChartInput{
+				ChartPath:   chart.ChartPath,
+				Namespace:   chart.Namespace,
+				ReleaseName: chart.ReleaseName,
+				RepoURL:     chart.RepoURL,
+				Values:      values,
+				Version:     version,
+				Timeout:     timeoutDuration,
+				Wait:        true,
+			}
+		}
+
+		lastUpgradedVersion := ""
+		for _, hook := range hooks {
+			if hook.PreUpgrade != nil {
+				common.LogVerbose(fmt.Sprintf("Running pre-upgrade hook %s@%s", chart.ReleaseName, hook.TargetVersion))
+				if err := hook.PreUpgrade(ctx, clientset, chart, installedRevision); err != nil {
+					return fmt.Errorf("Error running pre-upgrade hook for %s@%s: %w", chart.ReleaseName, hook.TargetVersion, err)
+				}
+			}
+
+			common.LogVerbose(fmt.Sprintf("Upgrading %s to intermediate version %s", chart.ReleaseName, hook.TargetVersion))
+			if err := helmAgent.InstallOrUpgradeChart(ctx, buildChartInput(hook.TargetVersion)); err != nil {
+				return fmt.Errorf("Error installing chart %s at version %s: %w", chart.ChartPath, hook.TargetVersion, err)
+			}
+			lastUpgradedVersion = hook.TargetVersion
+
+			if hook.PostUpgrade != nil {
+				common.LogVerbose(fmt.Sprintf("Running post-upgrade hook %s@%s", chart.ReleaseName, hook.TargetVersion))
+				if err := hook.PostUpgrade(ctx, clientset, chart, installedRevision); err != nil {
+					return fmt.Errorf("Error running post-upgrade hook for %s@%s: %w", chart.ReleaseName, hook.TargetVersion, err)
+				}
+			}
+		}
+
+		if lastUpgradedVersion == chart.Version {
+			continue
+		}
+
+		if err := helmAgent.InstallOrUpgradeChart(ctx, buildChartInput(chart.Version)); err != nil {
 			return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err)
 		}
 	}

plugins/scheduler-k3s/k8s.go

@@ -296,6 +296,55 @@ func (k KubernetesClient) CreateNamespace(ctx context.Context, input CreateNames
 	return *namespace, err
 }
 
+// DeleteDeploymentInput contains all the information needed to delete one or more Kubernetes deployments
+type DeleteDeploymentInput struct {
+	// Name is the Kubernetes deployment name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+}
+
+// DeleteDeployment deletes a Kubernetes deployment by name or label selector,
+// using foreground propagation so dependent ReplicaSets and Pods are removed
+// before the call returns.
+func (k KubernetesClient) DeleteDeployment(ctx context.Context, input DeleteDeploymentInput) error {
+	if input.Name == "" && input.LabelSelector == "" {
+		return fmt.Errorf("name or label selector is required")
+	}
+
+	if input.Name != "" && input.LabelSelector != "" {
+		return fmt.Errorf("name and label selector cannot be used together")
+	}
+
+	deleteOptions := metav1.DeleteOptions{
+		PropagationPolicy: ptr.To(metav1.DeletePropagationForeground),
+	}
+
+	if input.Name != "" {
+		return k.Client.AppsV1().Deployments(input.Namespace).Delete(ctx, input.Name, deleteOptions)
+	}
+
+	deployments, err := k.ListDeployments(ctx, ListDeploymentsInput{
+		Namespace:     input.Namespace,
+		LabelSelector: input.LabelSelector,
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing deployments: %w", err)
+	}
+
+	for _, deployment := range deployments {
+		if err := k.Client.AppsV1().Deployments(input.Namespace).Delete(ctx, deployment.Name, deleteOptions); err != nil {
+			return fmt.Errorf("Error deleting deployment %s: %w", deployment.Name, err)
+		}
+	}
+
+	return nil
+}
+
 // DeleteIngressInput contains all the information needed to delete a Kubernetes ingress
 type DeleteIngressInput struct {
 	// Name is the Kubernetes ingress name