From 8d0868770603f6acfa702ab403af8632220892a4 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 23 Nov 2024 02:04:54 -0500 Subject: [PATCH] feat: provide ability to customize chart values via `scheduler-k3s:set` Dokku ships with default charts and customizes their helm values. In certain cases, it may be useful to further customize those values to better support specific installations. With this change, it is now possible to set custom values by providing the key in the format `chart.$CHART_NAME.$PROPERTY`. For instance, to increase timeouts for the keda http addon: ```shell dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.tcpConnectTimeout 120s dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.keepAlive 120s dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.responseHeaderTimeout 120s dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.idleConnTimeout 120s dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.tlsHandshakeTimeout 120s dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.expectContinueTimeout 120s ``` To reset a value to its default, simply set the value to an empty string. ```shell dokku scheduler-k3s:set --global chart.keda-add-ons-http.interceptor.tcpConnectTimeout ``` Once the values are set, users will need to then run `scheduler-k3s:ensure-charts` with either the `--force` flag or specify the `--chart` flag with a list of charts to force apply. ```shell dokku scheduler-k3s:ensure-charts --force dokku scheduler-k3s:ensure-charts --chart keda-add-ons-http ``` Note that the existing dokku-provided values cannot be removed, but can be overridden. --- docs/deployment/schedulers/k3s.md | 30 +++++++++++++ plugins/scheduler-k3s/functions.go | 15 ++++++- plugins/scheduler-k3s/report.go | 11 +++++ .../src/subcommands/subcommands.go | 4 +- plugins/scheduler-k3s/subcommands.go | 45 ++++++++++++++++++- 5 files changed, 101 insertions(+), 4 deletions(-) diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 873406b5a..34a6106b7 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -572,6 +572,36 @@ dokku scheduler-k3s:set --global kube-context The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig. +### Customizing Helm Chart Properties + +Dokku includes a number of helm charts by default with settings that are optimized for Dokku. That said, it may be useful to further customize the charts for a given environment. Users can customize which charts are installed by setting properties prefixed with `chart.$CHART_NAME.` with the `--global` flag. + +```shell +dokku scheduler-k3s:set --global chart.cert-manager.version 1.13.3 +``` + +> [!NOTE] +> Properties follow dot-notation, and are expanded according to Helm's internal logic. See the [Helm documentation](https://helm.sh/docs/helm/helm_install/#helm-install) for `helm install` for further details. + + +To unset a chart property, omit the value from the `scheduler-k3s:set` call: + +```shell +dokku scheduler-k3s:set --global chart.cert-manager.version +``` + +A `scheduler-k3s:ensure-charts` command with the `--force` flag is required after changing any chart properties in order to have them apply. This will install all charts, not just the ones that have changed. + +```shell +dokku scheduler-k3s:ensure-charts --force +``` + +Alternatively, a comma separated list of chart names can be specified to only force install the specified charts: + +```shell +dokku scheduler-k3s:ensure-charts --force --chart-names cert-manager +``` + ## Scheduler Interface The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler. diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 4f84edca3..f93b910a4 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -23,6 +23,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/strvals" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -1493,7 +1494,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err) } - var values map[string]interface{} + values := map[string]interface{}{} if len(contents) > 0 { err = yaml.Unmarshal(contents, &values) if err != nil { @@ -1501,6 +1502,18 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn } } + chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart."+chart.ReleaseName+".") + if err != nil { + return fmt.Errorf("Error getting chart properties: %w", err) + } + for key, value := range chartProperties { + key = strings.TrimPrefix(key, "chart."+chart.ReleaseName+".") + strval := fmt.Sprintf("%s=%s", key, value) + if err := strvals.ParseInto(strval, values); err != nil { + return fmt.Errorf("Error parsing chart property %s: %w", strval, err) + } + } + helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter) if err != nil { return fmt.Errorf("Error creating helm agent: %w", err) diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index fd388c067..4e136eea5 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -37,6 +37,17 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure, } + chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart.") + if err != nil { + return fmt.Errorf("Unable to get property list: %w", err) + } + for name, value := range chartProperties { + flagName := "--scheduler-k3s-global-" + name + flags[flagName] = func(appName string) string { + return value + } + } + flagKeys := []string{} for flagKey := range flags { flagKeys = append(flagKeys, flagKey) diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 83c9e0db3..e23336bb8 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -75,8 +75,10 @@ func main() { err = scheduler_k3s.CommandClusterRemove(nodeName) case "ensure-charts": args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError) + forceInstall := args.Bool("force", false, "--force: force install all charts") + chartNames := args.StringSlice("charts", []string{}, "--charts: comma separated list of chart names to force install") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandEnsureCharts() + err = scheduler_k3s.CommandEnsureCharts(*forceInstall, *chartNames) case "initialize": args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 51468339c..4eb30f854 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "os/signal" + "slices" "sort" "strings" "syscall" @@ -799,7 +800,7 @@ func CommandClusterRemove(nodeName string) error { } // CommandEnsureCharts ensures that the required helm charts are installed -func CommandEnsureCharts() error { +func CommandEnsureCharts(forceInstall bool, forceChartNames []string) error { ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGHUP, @@ -844,6 +845,16 @@ func CommandEnsureCharts() error { return false } + if forceInstall { + common.LogVerbose("Force installing chart") + return true + } + + if len(forceChartNames) > 0 && slices.Contains(forceChartNames, chart.ReleaseName) { + common.LogVerbose("Force installing chart due to flag") + return true + } + helmAgent := namespacedHelmAgents[chart.Namespace] latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName) if err != nil { @@ -944,7 +955,37 @@ func CommandReport(appName string, format string, infoFlag string) error { // CommandSet set or clear a scheduler-k3s property for an app func CommandSet(appName string, property string, value string) error { - common.CommandPropertySet("scheduler-k3s", appName, property, value, DefaultProperties, GlobalProperties) + validProperties := DefaultProperties + globalProperties := GlobalProperties + if strings.HasPrefix(property, "chart.") { + if appName != "--global" { + return fmt.Errorf("Chart properties can only be set globally") + } + + chartParts := strings.SplitN(property, ".", 3) + if len(chartParts) != 3 { + return fmt.Errorf("Invalid chart property, expected format: chart.$CHART_NAME.$PROPERTY: %s", property) + } + + if chartParts[1] == "" { + return fmt.Errorf("Invalid chart property, missing chart name") + } + + chartName := chartParts[1] + for _, chart := range HelmCharts { + if chart.ReleaseName == chartName { + validProperties[property] = "" + globalProperties[property] = true + break + } + } + + if _, ok := validProperties[property]; !ok { + return fmt.Errorf("Invalid chart property, no matching chart found: %s", property) + } + } + + common.CommandPropertySet("scheduler-k3s", appName, property, value, validProperties, globalProperties) letsencryptProperties := map[string]bool{ "letsencrypt-email-prod": true,