Merge pull request #7379 from dokku/scheduler-k3s-charts-set

docs/deployment/schedulers/k3s.md

@@ -572,6 +572,36 @@ dokku scheduler-k3s:set --global kube-context
 
 The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig.
 
+### Customizing Helm Chart Properties
+
+Dokku includes a number of helm charts by default with settings that are optimized for Dokku. That said, it may be useful to further customize the charts for a given environment. Users can customize which charts are installed by setting properties prefixed with `chart.$CHART_NAME.` with the `--global` flag.
+
+```shell
+dokku scheduler-k3s:set --global chart.cert-manager.version 1.13.3
+```
+
+> [!NOTE]
+> Properties follow dot-notation, and are expanded according to Helm's internal logic. See the [Helm documentation](https://helm.sh/docs/helm/helm_install/#helm-install) for `helm install` for further details.
+
+
+To unset a chart property, omit the value from the `scheduler-k3s:set` call:
+
+```shell
+dokku scheduler-k3s:set --global chart.cert-manager.version
+```
+
+A `scheduler-k3s:ensure-charts` command with the `--force` flag is required after changing any chart properties in order to have them apply. This will install all charts, not just the ones that have changed.
+
+```shell
+dokku scheduler-k3s:ensure-charts --force
+```
+
+Alternatively, a comma separated list of chart names can be specified to only force install the specified charts:
+
+```shell
+dokku scheduler-k3s:ensure-charts --force --chart-names cert-manager
+```
+
 ## Scheduler Interface
 
 The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler.

plugins/scheduler-k3s/functions.go

@@ -23,6 +23,7 @@ import (
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/yaml.v3"
+	"helm.sh/helm/v3/pkg/strvals"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -1493,7 +1494,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 			return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err)
 		}
 
-		var values map[string]interface{}
+		values := map[string]interface{}{}
 		if len(contents) > 0 {
 			err = yaml.Unmarshal(contents, &values)
 			if err != nil {
@@ -1501,6 +1502,18 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 			}
 		}
 
+		chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart."+chart.ReleaseName+".")
+		if err != nil {
+			return fmt.Errorf("Error getting chart properties: %w", err)
+		}
+		for key, value := range chartProperties {
+			key = strings.TrimPrefix(key, "chart."+chart.ReleaseName+".")
+			strval := fmt.Sprintf("%s=%s", key, value)
+			if err := strvals.ParseInto(strval, values); err != nil {
+				return fmt.Errorf("Error parsing chart property %s: %w", strval, err)
+			}
+		}
+
 		helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter)
 		if err != nil {
 			return fmt.Errorf("Error creating helm agent: %w", err)

plugins/scheduler-k3s/report.go

@@ -37,6 +37,17 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		"--scheduler-k3s-global-rollback-on-failure":    reportGlobalRollbackOnFailure,
 	}
 
+	chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart.")
+	if err != nil {
+		return fmt.Errorf("Unable to get property list: %w", err)
+	}
+	for name, value := range chartProperties {
+		flagName := "--scheduler-k3s-global-" + name
+		flags[flagName] = func(appName string) string {
+			return value
+		}
+	}
+
 	flagKeys := []string{}
 	for flagKey := range flags {
 		flagKeys = append(flagKeys, flagKey)

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -75,8 +75,10 @@ func main() {
 		err = scheduler_k3s.CommandClusterRemove(nodeName)
 	case "ensure-charts":
 		args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError)
+		forceInstall := args.Bool("force", false, "--force: force install all charts")
+		chartNames := args.StringSlice("charts", []string{}, "--charts: comma separated list of chart names to force install")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandEnsureCharts()
+		err = scheduler_k3s.CommandEnsureCharts(*forceInstall, *chartNames)
 	case "initialize":
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")

plugins/scheduler-k3s/subcommands.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"slices"
 	"sort"
 	"strings"
 	"syscall"
@@ -799,7 +800,7 @@ func CommandClusterRemove(nodeName string) error {
 }
 
 // CommandEnsureCharts ensures that the required helm charts are installed
-func CommandEnsureCharts() error {
+func CommandEnsureCharts(forceInstall bool, forceChartNames []string) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -844,6 +845,16 @@ func CommandEnsureCharts() error {
 			return false
 		}
 
+		if forceInstall {
+			common.LogVerbose("Force installing chart")
+			return true
+		}
+
+		if len(forceChartNames) > 0 && slices.Contains(forceChartNames, chart.ReleaseName) {
+			common.LogVerbose("Force installing chart due to flag")
+			return true
+		}
+
 		helmAgent := namespacedHelmAgents[chart.Namespace]
 		latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName)
 		if err != nil {
@@ -944,7 +955,37 @@ func CommandReport(appName string, format string, infoFlag string) error {
 
 // CommandSet set or clear a scheduler-k3s property for an app
 func CommandSet(appName string, property string, value string) error {
-	common.CommandPropertySet("scheduler-k3s", appName, property, value, DefaultProperties, GlobalProperties)
+	validProperties := DefaultProperties
+	globalProperties := GlobalProperties
+	if strings.HasPrefix(property, "chart.") {
+		if appName != "--global" {
+			return fmt.Errorf("Chart properties can only be set globally")
+		}
+
+		chartParts := strings.SplitN(property, ".", 3)
+		if len(chartParts) != 3 {
+			return fmt.Errorf("Invalid chart property, expected format: chart.$CHART_NAME.$PROPERTY: %s", property)
+		}
+
+		if chartParts[1] == "" {
+			return fmt.Errorf("Invalid chart property, missing chart name")
+		}
+
+		chartName := chartParts[1]
+		for _, chart := range HelmCharts {
+			if chart.ReleaseName == chartName {
+				validProperties[property] = ""
+				globalProperties[property] = true
+				break
+			}
+		}
+
+		if _, ok := validProperties[property]; !ok {
+			return fmt.Errorf("Invalid chart property, no matching chart found: %s", property)
+		}
+	}
+
+	common.CommandPropertySet("scheduler-k3s", appName, property, value, validProperties, globalProperties)
 
 	letsencryptProperties := map[string]bool{
 		"letsencrypt-email-prod": true,