diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 873406b5a..34a6106b7 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -572,6 +572,36 @@ dokku scheduler-k3s:set --global kube-context The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig. +### Customizing Helm Chart Properties + +Dokku includes a number of helm charts by default with settings that are optimized for Dokku. That said, it may be useful to further customize the charts for a given environment. Users can customize which charts are installed by setting properties prefixed with `chart.$CHART_NAME.` with the `--global` flag. + +```shell +dokku scheduler-k3s:set --global chart.cert-manager.version 1.13.3 +``` + +> [!NOTE] +> Properties follow dot-notation, and are expanded according to Helm's internal logic. See the [Helm documentation](https://helm.sh/docs/helm/helm_install/#helm-install) for `helm install` for further details. + + +To unset a chart property, omit the value from the `scheduler-k3s:set` call: + +```shell +dokku scheduler-k3s:set --global chart.cert-manager.version +``` + +A `scheduler-k3s:ensure-charts` command with the `--force` flag is required after changing any chart properties in order to have them apply. This will install all charts, not just the ones that have changed. + +```shell +dokku scheduler-k3s:ensure-charts --force +``` + +Alternatively, a comma separated list of chart names can be specified to only force install the specified charts: + +```shell +dokku scheduler-k3s:ensure-charts --force --chart-names cert-manager +``` + ## Scheduler Interface The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler. diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 4f84edca3..f93b910a4 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -23,6 +23,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/strvals" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -1493,7 +1494,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err) } - var values map[string]interface{} + values := map[string]interface{}{} if len(contents) > 0 { err = yaml.Unmarshal(contents, &values) if err != nil { @@ -1501,6 +1502,18 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn } } + chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart."+chart.ReleaseName+".") + if err != nil { + return fmt.Errorf("Error getting chart properties: %w", err) + } + for key, value := range chartProperties { + key = strings.TrimPrefix(key, "chart."+chart.ReleaseName+".") + strval := fmt.Sprintf("%s=%s", key, value) + if err := strvals.ParseInto(strval, values); err != nil { + return fmt.Errorf("Error parsing chart property %s: %w", strval, err) + } + } + helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter) if err != nil { return fmt.Errorf("Error creating helm agent: %w", err) diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index fd388c067..4e136eea5 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -37,6 +37,17 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure, } + chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart.") + if err != nil { + return fmt.Errorf("Unable to get property list: %w", err) + } + for name, value := range chartProperties { + flagName := "--scheduler-k3s-global-" + name + flags[flagName] = func(appName string) string { + return value + } + } + flagKeys := []string{} for flagKey := range flags { flagKeys = append(flagKeys, flagKey) diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 83c9e0db3..e23336bb8 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -75,8 +75,10 @@ func main() { err = scheduler_k3s.CommandClusterRemove(nodeName) case "ensure-charts": args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError) + forceInstall := args.Bool("force", false, "--force: force install all charts") + chartNames := args.StringSlice("charts", []string{}, "--charts: comma separated list of chart names to force install") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandEnsureCharts() + err = scheduler_k3s.CommandEnsureCharts(*forceInstall, *chartNames) case "initialize": args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 51468339c..4eb30f854 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "os/signal" + "slices" "sort" "strings" "syscall" @@ -799,7 +800,7 @@ func CommandClusterRemove(nodeName string) error { } // CommandEnsureCharts ensures that the required helm charts are installed -func CommandEnsureCharts() error { +func CommandEnsureCharts(forceInstall bool, forceChartNames []string) error { ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGHUP, @@ -844,6 +845,16 @@ func CommandEnsureCharts() error { return false } + if forceInstall { + common.LogVerbose("Force installing chart") + return true + } + + if len(forceChartNames) > 0 && slices.Contains(forceChartNames, chart.ReleaseName) { + common.LogVerbose("Force installing chart due to flag") + return true + } + helmAgent := namespacedHelmAgents[chart.Namespace] latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName) if err != nil { @@ -944,7 +955,37 @@ func CommandReport(appName string, format string, infoFlag string) error { // CommandSet set or clear a scheduler-k3s property for an app func CommandSet(appName string, property string, value string) error { - common.CommandPropertySet("scheduler-k3s", appName, property, value, DefaultProperties, GlobalProperties) + validProperties := DefaultProperties + globalProperties := GlobalProperties + if strings.HasPrefix(property, "chart.") { + if appName != "--global" { + return fmt.Errorf("Chart properties can only be set globally") + } + + chartParts := strings.SplitN(property, ".", 3) + if len(chartParts) != 3 { + return fmt.Errorf("Invalid chart property, expected format: chart.$CHART_NAME.$PROPERTY: %s", property) + } + + if chartParts[1] == "" { + return fmt.Errorf("Invalid chart property, missing chart name") + } + + chartName := chartParts[1] + for _, chart := range HelmCharts { + if chart.ReleaseName == chartName { + validProperties[property] = "" + globalProperties[property] = true + break + } + } + + if _, ok := validProperties[property]; !ok { + return fmt.Errorf("Invalid chart property, no matching chart found: %s", property) + } + } + + common.CommandPropertySet("scheduler-k3s", appName, property, value, validProperties, globalProperties) letsencryptProperties := map[string]bool{ "letsencrypt-email-prod": true,