feat: provide ability to customize chart values via scheduler-k3s:set

Dokku ships with default charts and customizes their helm values. In certain cases, it may be useful to further customize those values to better support specific installations. With this change, it is now possible to set custom values by providing the key in the format `chart.$CHART_NAME.$PROPERTY`. For instance, to increase timeouts for the keda http addon:

```shell
dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.tcpConnectTimeout 120s
dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.keepAlive 120s
dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.responseHeaderTimeout 120s
dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.idleConnTimeout 120s
dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.tlsHandshakeTimeout 120s
dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.expectContinueTimeout 120s
```

To reset a value to its default, simply set the value to an empty string.

```shell
dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.tcpConnectTimeout
```

Once the values are set, users will need to then run `scheduler-k3s:ensure-charts` with either the `--force` flag or specify the `--chart` flag with a list of charts to force apply.

```shell
dokku scheduler-k3s:ensure-charts --force

dokku scheduler-k3s:ensure-charts --chart keda-add-ons-http
```

Note that the existing dokku-provided values cannot be removed, but can be overridden.

docs/deployment/schedulers/k3s.md

@@ -572,6 +572,36 @@ dokku scheduler-k3s:set --global kube-context
 
 The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig.
 
+### Customizing Helm Chart Properties
+
+Dokku includes a number of helm charts by default with settings that are optimized for Dokku. That said, it may be useful to further customize the charts for a given environment. Users can customize which charts are installed by setting properties prefixed with `chart.$CHART_NAME.` with the `--global` flag.
+
+```shell
+dokku scheduler-k3s:set --global chart.cert-manager.version 1.13.3
+```
+
+> [!NOTE]
+> Properties follow dot-notation, and are expanded according to Helm's internal logic. See the [Helm documentation](https://helm.sh/docs/helm/helm_install/#helm-install) for `helm install` for further details.
+
+
+To unset a chart property, omit the value from the `scheduler-k3s:set` call:
+
+```shell
+dokku scheduler-k3s:set --global chart.cert-manager.version
+```
+
+A `scheduler-k3s:ensure-charts` command with the `--force` flag is required after changing any chart properties in order to have them apply. This will install all charts, not just the ones that have changed.
+
+```shell
+dokku scheduler-k3s:ensure-charts --force
+```
+
+Alternatively, a comma separated list of chart names can be specified to only force install the specified charts:
+
+```shell
+dokku scheduler-k3s:ensure-charts --force --chart-names cert-manager
+```
+
 ## Scheduler Interface
 
 The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler.

plugins/scheduler-k3s/functions.go

@@ -23,6 +23,7 @@ import (
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/yaml.v3"
+	"helm.sh/helm/v3/pkg/strvals"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -1493,7 +1494,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 			return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err)
 		}
 
-		var values map[string]interface{}
+		values := map[string]interface{}{}
 		if len(contents) > 0 {
 			err = yaml.Unmarshal(contents, &values)
 			if err != nil {
@@ -1501,6 +1502,18 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 			}
 		}
 
+		chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart."+chart.ReleaseName+".")
+		if err != nil {
+			return fmt.Errorf("Error getting chart properties: %w", err)
+		}
+		for key, value := range chartProperties {
+			key = strings.TrimPrefix(key, "chart."+chart.ReleaseName+".")
+			strval := fmt.Sprintf("%s=%s", key, value)
+			if err := strvals.ParseInto(strval, values); err != nil {
+				return fmt.Errorf("Error parsing chart property %s: %w", strval, err)
+			}
+		}
+
 		helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter)
 		if err != nil {
 			return fmt.Errorf("Error creating helm agent: %w", err)

plugins/scheduler-k3s/report.go

@@ -37,6 +37,17 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		"--scheduler-k3s-global-rollback-on-failure":    reportGlobalRollbackOnFailure,
 	}
 
+	chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart.")
+	if err != nil {
+		return fmt.Errorf("Unable to get property list: %w", err)
+	}
+	for name, value := range chartProperties {
+		flagName := "--scheduler-k3s-global-" + name
+		flags[flagName] = func(appName string) string {
+			return value
+		}
+	}
+
 	flagKeys := []string{}
 	for flagKey := range flags {
 		flagKeys = append(flagKeys, flagKey)

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -75,8 +75,10 @@ func main() {
 		err = scheduler_k3s.CommandClusterRemove(nodeName)
 	case "ensure-charts":
 		args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError)
+		forceInstall := args.Bool("force", false, "--force: force install all charts")
+		chartNames := args.StringSlice("charts", []string{}, "--charts: comma separated list of chart names to force install")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandEnsureCharts()
+		err = scheduler_k3s.CommandEnsureCharts(*forceInstall, *chartNames)
 	case "initialize":
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")

plugins/scheduler-k3s/subcommands.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"slices"
 	"sort"
 	"strings"
 	"syscall"
@@ -799,7 +800,7 @@ func CommandClusterRemove(nodeName string) error {
 }
 
 // CommandEnsureCharts ensures that the required helm charts are installed
-func CommandEnsureCharts() error {
+func CommandEnsureCharts(forceInstall bool, forceChartNames []string) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -844,6 +845,16 @@ func CommandEnsureCharts() error {
 			return false
 		}
 
+		if forceInstall {
+			common.LogVerbose("Force installing chart")
+			return true
+		}
+
+		if len(forceChartNames) > 0 && slices.Contains(forceChartNames, chart.ReleaseName) {
+			common.LogVerbose("Force installing chart due to flag")
+			return true
+		}
+
 		helmAgent := namespacedHelmAgents[chart.Namespace]
 		latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName)
 		if err != nil {
@@ -944,7 +955,37 @@ func CommandReport(appName string, format string, infoFlag string) error {
 
 // CommandSet set or clear a scheduler-k3s property for an app
 func CommandSet(appName string, property string, value string) error {
-	common.CommandPropertySet("scheduler-k3s", appName, property, value, DefaultProperties, GlobalProperties)
+	validProperties := DefaultProperties
+	globalProperties := GlobalProperties
+	if strings.HasPrefix(property, "chart.") {
+		if appName != "--global" {
+			return fmt.Errorf("Chart properties can only be set globally")
+		}
+
+		chartParts := strings.SplitN(property, ".", 3)
+		if len(chartParts) != 3 {
+			return fmt.Errorf("Invalid chart property, expected format: chart.$CHART_NAME.$PROPERTY: %s", property)
+		}
+
+		if chartParts[1] == "" {
+			return fmt.Errorf("Invalid chart property, missing chart name")
+		}
+
+		chartName := chartParts[1]
+		for _, chart := range HelmCharts {
+			if chart.ReleaseName == chartName {
+				validProperties[property] = ""
+				globalProperties[property] = true
+				break
+			}
+		}
+
+		if _, ok := validProperties[property]; !ok {
+			return fmt.Errorf("Invalid chart property, no matching chart found: %s", property)
+		}
+	}
+
+	common.CommandPropertySet("scheduler-k3s", appName, property, value, validProperties, globalProperties)
 
 	letsencryptProperties := map[string]bool{
 		"letsencrypt-email-prod": true,