feat: add version-aware helm chart upgrade callbacks

Some helm chart upgrades require side-effects that a plain `helm upgrade` cannot perform, such as deleting deployments whose immutable selectors changed. Charts may now register pre and post upgrade hooks against a target version; applicable hooks fire in ascending semver order, each bracketed around an upgrade to its version, with a final upgrade to the chart's configured version when needed. The new `keda-add-ons-http` 0.12.2 hook deletes the chart-managed deployments before the upgrade so the new selectors take effect cleanly.

plugins/scheduler-k3s/chart_hooks.go

@@ -0,0 +1,104 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/Masterminds/semver/v3"
+)
+
+// ChartHookFunc is the signature for a function that runs around a helm chart
+// upgrade. It receives the active Kubernetes client, the chart definition, and
+// the helm release that was installed before the upgrade started.
+type ChartHookFunc func(ctx context.Context, clientset KubernetesClient, chart HelmChart, installedRevision Release) error
+
+// ChartHook attaches version-specific behaviour to a helm chart upgrade. A
+// hook fires when the currently installed release version is below
+// TargetVersion and the chart's configured target version is at or above
+// TargetVersion. PreUpgrade runs before the chart is upgraded to
+// TargetVersion; PostUpgrade runs after that intermediate upgrade succeeds.
+type ChartHook struct {
+	TargetVersion string
+	PreUpgrade    ChartHookFunc
+	PostUpgrade   ChartHookFunc
+}
+
+// ChartHooks is keyed by HelmChart.ReleaseName. selectChartHooks resorts
+// entries by semver, so authoring order is informative rather than load-bearing.
+var ChartHooks = map[string][]ChartHook{
+	"keda-add-ons-http": {
+		{
+			TargetVersion: "0.12.2",
+			PreUpgrade:    deleteKedaAddOnsHttpDeployments,
+		},
+	},
+}
+
+// selectChartHooks returns the hooks for releaseName whose TargetVersion is
+// strictly greater than installedVersion and less than or equal to
+// targetVersion, sorted ascending by TargetVersion. An empty installedVersion
+// signals a fresh install and yields no hooks.
+func selectChartHooks(releaseName, installedVersion, targetVersion string) ([]ChartHook, error) {
+	if installedVersion == "" {
+		return nil, nil
+	}
+
+	hooks, ok := ChartHooks[releaseName]
+	if !ok || len(hooks) == 0 {
+		return nil, nil
+	}
+
+	installed, err := semver.NewVersion(installedVersion)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid installed version %q for chart %s: %w", installedVersion, releaseName, err)
+	}
+
+	target, err := semver.NewVersion(targetVersion)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid target version %q for chart %s: %w", targetVersion, releaseName, err)
+	}
+
+	type indexedHook struct {
+		version *semver.Version
+		hook    ChartHook
+	}
+
+	selected := []indexedHook{}
+	for _, hook := range hooks {
+		hookVersion, err := semver.NewVersion(hook.TargetVersion)
+		if err != nil {
+			return nil, fmt.Errorf("Invalid hook TargetVersion %q for chart %s: %w", hook.TargetVersion, releaseName, err)
+		}
+
+		if hookVersion.Compare(installed) <= 0 {
+			continue
+		}
+		if hookVersion.Compare(target) > 0 {
+			continue
+		}
+
+		selected = append(selected, indexedHook{version: hookVersion, hook: hook})
+	}
+
+	sort.Slice(selected, func(i, j int) bool {
+		return selected[i].version.LessThan(selected[j].version)
+	})
+
+	out := make([]ChartHook, 0, len(selected))
+	for _, entry := range selected {
+		out = append(out, entry.hook)
+	}
+	return out, nil
+}
+
+// deleteKedaAddOnsHttpDeployments removes the deployments managed by the
+// keda-add-ons-http helm release so the chart can recreate them with the
+// updated selectors introduced in 0.12.x. Selectors on Deployments are
+// immutable, so a straight helm upgrade fails without this step.
+func deleteKedaAddOnsHttpDeployments(ctx context.Context, clientset KubernetesClient, chart HelmChart, installedRevision Release) error {
+	return clientset.DeleteDeployment(ctx, DeleteDeploymentInput{
+		Namespace:     chart.Namespace,
+		LabelSelector: "app.kubernetes.io/instance=keda-add-ons-http",
+	})
+}

plugins/scheduler-k3s/chart_hooks_test.go

@@ -0,0 +1,159 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"testing"
+)
+
+func makeHook(version string) ChartHook {
+	noop := func(context.Context, KubernetesClient, HelmChart, Release) error { return nil }
+	return ChartHook{
+		TargetVersion: version,
+		PreUpgrade:    noop,
+	}
+}
+
+func versionsOf(hooks []ChartHook) []string {
+	out := make([]string, 0, len(hooks))
+	for _, hook := range hooks {
+		out = append(out, hook.TargetVersion)
+	}
+	return out
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func TestSelectChartHooks(t *testing.T) {
+	previous := ChartHooks
+	t.Cleanup(func() { ChartHooks = previous })
+
+	ChartHooks = map[string][]ChartHook{
+		"chart-with-hooks": {
+			makeHook("0.12.0"),
+			makeHook("0.10.0"),
+			makeHook("0.11.0"),
+		},
+	}
+
+	cases := []struct {
+		name             string
+		releaseName      string
+		installedVersion string
+		targetVersion    string
+		want             []string
+		wantErr          bool
+	}{
+		{
+			name:             "fresh install yields no hooks",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "",
+			targetVersion:    "0.12.0",
+			want:             nil,
+		},
+		{
+			name:             "no hooks registered for release",
+			releaseName:      "chart-without-hooks",
+			installedVersion: "0.5.0",
+			targetVersion:    "0.6.0",
+			want:             nil,
+		},
+		{
+			name:             "single applicable hook returned",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.10.0",
+			targetVersion:    "0.11.0",
+			want:             []string{"0.11.0"},
+		},
+		{
+			name:             "all hooks below installed version excluded",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.12.0",
+			targetVersion:    "0.12.0",
+			want:             nil,
+		},
+		{
+			name:             "all applicable hooks returned in ascending semver order",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.9.0",
+			targetVersion:    "0.12.0",
+			want:             []string{"0.10.0", "0.11.0", "0.12.0"},
+		},
+		{
+			name:             "partial coverage returns only unsatisfied hooks",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.10.0",
+			targetVersion:    "0.12.0",
+			want:             []string{"0.11.0", "0.12.0"},
+		},
+		{
+			name:             "target above all hooks still caps at chart target",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.9.0",
+			targetVersion:    "0.11.0",
+			want:             []string{"0.10.0", "0.11.0"},
+		},
+		{
+			name:             "unparseable installed version errors",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "not-a-version",
+			targetVersion:    "0.12.0",
+			wantErr:          true,
+		},
+		{
+			name:             "unparseable target version errors",
+			releaseName:      "chart-with-hooks",
+			installedVersion: "0.9.0",
+			targetVersion:    "not-a-version",
+			wantErr:          true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := selectChartHooks(tc.releaseName, tc.installedVersion, tc.targetVersion)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("expected error, got nil")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			gotVersions := versionsOf(got)
+			if !equalStrings(gotVersions, tc.want) {
+				t.Fatalf("versions mismatch: got %v, want %v", gotVersions, tc.want)
+			}
+		})
+	}
+}
+
+func TestSelectChartHooksTreatsLeadingVAsEqual(t *testing.T) {
+	previous := ChartHooks
+	t.Cleanup(func() { ChartHooks = previous })
+
+	ChartHooks = map[string][]ChartHook{
+		"chart-v-prefixed": {
+			makeHook("v1.13.3"),
+		},
+	}
+
+	got, err := selectChartHooks("chart-v-prefixed", "v1.12.0", "v1.13.3")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(got) != 1 || got[0].TargetVersion != "v1.13.3" {
+		t.Fatalf("expected v1.13.3 hook, got %v", versionsOf(got))
+	}
+}

plugins/scheduler-k3s/functions.go

@@ -1679,17 +1679,57 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 			return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
 		}
 
-		err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
-			ChartPath:   chart.ChartPath,
-			Namespace:   chart.Namespace,
-			ReleaseName: chart.ReleaseName,
-			RepoURL:     chart.RepoURL,
-			Values:      values,
-			Version:     chart.Version,
-			Timeout:     timeoutDuration,
-			Wait:        true,
-		})
+		installedRevision, err := helmAgent.InstalledRevision(chart.ReleaseName)
 		if err != nil {
+			return fmt.Errorf("Error getting installed revision for %s: %w", chart.ReleaseName, err)
+		}
+
+		hooks, err := selectChartHooks(chart.ReleaseName, installedRevision.Version, chart.Version)
+		if err != nil {
+			return fmt.Errorf("Error selecting upgrade hooks for chart %s: %w", chart.ReleaseName, err)
+		}
+
+		buildChartInput := func(version string) ChartInput {
+			return ChartInput{
+				ChartPath:   chart.ChartPath,
+				Namespace:   chart.Namespace,
+				ReleaseName: chart.ReleaseName,
+				RepoURL:     chart.RepoURL,
+				Values:      values,
+				Version:     version,
+				Timeout:     timeoutDuration,
+				Wait:        true,
+			}
+		}
+
+		lastUpgradedVersion := ""
+		for _, hook := range hooks {
+			if hook.PreUpgrade != nil {
+				common.LogVerbose(fmt.Sprintf("Running pre-upgrade hook %s@%s", chart.ReleaseName, hook.TargetVersion))
+				if err := hook.PreUpgrade(ctx, clientset, chart, installedRevision); err != nil {
+					return fmt.Errorf("Error running pre-upgrade hook for %s@%s: %w", chart.ReleaseName, hook.TargetVersion, err)
+				}
+			}
+
+			common.LogVerbose(fmt.Sprintf("Upgrading %s to intermediate version %s", chart.ReleaseName, hook.TargetVersion))
+			if err := helmAgent.InstallOrUpgradeChart(ctx, buildChartInput(hook.TargetVersion)); err != nil {
+				return fmt.Errorf("Error installing chart %s at version %s: %w", chart.ChartPath, hook.TargetVersion, err)
+			}
+			lastUpgradedVersion = hook.TargetVersion
+
+			if hook.PostUpgrade != nil {
+				common.LogVerbose(fmt.Sprintf("Running post-upgrade hook %s@%s", chart.ReleaseName, hook.TargetVersion))
+				if err := hook.PostUpgrade(ctx, clientset, chart, installedRevision); err != nil {
+					return fmt.Errorf("Error running post-upgrade hook for %s@%s: %w", chart.ReleaseName, hook.TargetVersion, err)
+				}
+			}
+		}
+
+		if lastUpgradedVersion == chart.Version {
+			continue
+		}
+
+		if err := helmAgent.InstallOrUpgradeChart(ctx, buildChartInput(chart.Version)); err != nil {
 			return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err)
 		}
 	}

plugins/scheduler-k3s/k8s.go

@@ -296,6 +296,55 @@ func (k KubernetesClient) CreateNamespace(ctx context.Context, input CreateNames
 	return *namespace, err
 }
 
+// DeleteDeploymentInput contains all the information needed to delete one or more Kubernetes deployments
+type DeleteDeploymentInput struct {
+	// Name is the Kubernetes deployment name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+}
+
+// DeleteDeployment deletes a Kubernetes deployment by name or label selector,
+// using foreground propagation so dependent ReplicaSets and Pods are removed
+// before the call returns.
+func (k KubernetesClient) DeleteDeployment(ctx context.Context, input DeleteDeploymentInput) error {
+	if input.Name == "" && input.LabelSelector == "" {
+		return fmt.Errorf("name or label selector is required")
+	}
+
+	if input.Name != "" && input.LabelSelector != "" {
+		return fmt.Errorf("name and label selector cannot be used together")
+	}
+
+	deleteOptions := metav1.DeleteOptions{
+		PropagationPolicy: ptr.To(metav1.DeletePropagationForeground),
+	}
+
+	if input.Name != "" {
+		return k.Client.AppsV1().Deployments(input.Namespace).Delete(ctx, input.Name, deleteOptions)
+	}
+
+	deployments, err := k.ListDeployments(ctx, ListDeploymentsInput{
+		Namespace:     input.Namespace,
+		LabelSelector: input.LabelSelector,
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing deployments: %w", err)
+	}
+
+	for _, deployment := range deployments {
+		if err := k.Client.AppsV1().Deployments(input.Namespace).Delete(ctx, deployment.Name, deleteOptions); err != nil {
+			return fmt.Errorf("Error deleting deployment %s: %w", deployment.Name, err)
+		}
+	}
+
+	return nil
+}
+
 // DeleteIngressInput contains all the information needed to delete a Kubernetes ingress
 type DeleteIngressInput struct {
 	// Name is the Kubernetes ingress name