Merge pull request #7776 from dokku/k3s-kustomize

Add kustomize support for applying helm charts

docs/deployment/schedulers/k3s.md

@@ -141,6 +141,7 @@ The k3s plugin provides a number of settings that can be used to managed deploym
 | Name                  | Description                                       | Global Default     |
 |-----------------------|---------------------------------------------------|--------------------|
 | `deploy-timeout`      | Controls when app deploys will timeout in seconds | `300s`             |
+| `kustomize-root-path` | Controls the folder context from the deployed repository used for Kustomize | `config/kustomize` |
 | `image-pull-secrets`  | Name of a kubernetes secret used to auth against a registry | Contents of `~/.docker/config.json` from Dokku server |
 | `namespace`           | Controls the namespace used for resource creation | `default`          |
 | `rollback-on-failure` | Whether to rollback failed deploys                | `false`            |
@@ -365,7 +366,6 @@ dokku scheduler-k3s:ensure-charts
 
 Once the chart is configured, an `http` trigger can be specified like so:
 
-
 ```json
 {
     "formation": {
@@ -461,6 +461,30 @@ dokku scheduler-k3s:autoscaling-auth:report node-js-app --include-metadata
       Datadog datadogSite:           us5.datadoghq.com
 ```
 
+### Integrating Kustomize
+
+Dokku supports integration with [Kustomize](https://kustomize.io/) to further customize the generated helm charts for app deployments. For example, a `config/kustomize/kustomization.yaml` file with the following contents will override the scale for each process deployed to `3`:
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- rendered.yaml
+patches:
+- patch: |-
+    - op: replace
+      path: /spec/replicas
+      value: 3
+  target:
+      group: apps
+      version: v1
+      kind: Deployment
+```
+
+When using Kustomize with Dokku, all Kustomize-files _must_ be placed in the `config/kustomize` folder, with a `kustomization.yaml` file being the entrypoint to Kustomize. Dokku will render the helm chart to a `rendered.yaml` file, and then execute Kustomize with the `config/kustomize` folder as the context.
+
+See the [Kustomize](https://kustomize.io/) website for more details on how to use Kustomize.
+
 ### Using kubectl remotely
 
 > [!WARNING]
@@ -515,7 +539,6 @@ dokku scheduler-k3s:set --global chart.cert-manager.version 1.13.3
 > [!NOTE]
 > Properties follow dot-notation, and are expanded according to Helm's internal logic. See the [Helm documentation](https://helm.sh/docs/helm/helm_install/#helm-install) for `helm install` for further details.
 
-
 To unset a chart property, omit the value from the `scheduler-k3s:set` call:
 
 ```shell

plugins/scheduler-k3s/Makefile

@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
+TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 

plugins/scheduler-k3s/functions.go

@@ -576,7 +576,7 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) {
 	}
 
 	if !ok {
-		common.LogWarn(fmt.Sprintf("No autoscaling config found for %s", input.AppName))
+		common.LogWarn(fmt.Sprintf("No autoscaling config found for %s:%s", input.AppName, input.ProcessType))
 		return ProcessAutoscaling{}, nil
 	}
 
@@ -1190,6 +1190,28 @@ func getGlobalLetsencryptEmailStag() string {
 	return common.PropertyGetDefault("scheduler-k3s", "--global", "letsencrypt-email-stag", "")
 }
 
+func getKustomizeDirectory(appName string) string {
+	directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName)
+	return filepath.Join(directory, "kustomization")
+}
+
+func getKustomizeRootPath(appName string) string {
+	return common.PropertyGetDefault("scheduler-k3s", appName, "kustomize-root-path", "")
+}
+
+func getGlobalKustomizeRootPath() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "kustomize-root-path", "config/kustomize")
+}
+
+func getComputedKustomizeRootPath(appName string) string {
+	kustomizeRootPath := getKustomizeRootPath(appName)
+	if kustomizeRootPath == "" {
+		kustomizeRootPath = getGlobalKustomizeRootPath()
+	}
+
+	return kustomizeRootPath
+}
+
 func getNamespace(appName string) string {
 	return common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "")
 }
@@ -1477,6 +1499,34 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
+func getProcessSpecificKustomizeRootPath(appName string) string {
+	if !hasKustomizeDirectory(appName) {
+		return ""
+	}
+
+	directory := getKustomizeDirectory(appName)
+	processSpecificDirectory := fmt.Sprintf("%s.%s", directory, os.Getenv("DOKKU_PID"))
+	if common.DirectoryExists(processSpecificDirectory) {
+		return processSpecificDirectory
+	}
+
+	return directory
+}
+
+func hasKustomizeDirectory(appName string) bool {
+	directory := getKustomizeDirectory(appName)
+
+	if common.DirectoryExists(fmt.Sprintf("%s.%s.missing", directory, os.Getenv("DOKKU_PID"))) {
+		return false
+	}
+
+	if common.DirectoryExists(fmt.Sprintf("%s.%s", directory, os.Getenv("DOKKU_PID"))) {
+		return true
+	}
+
+	return common.DirectoryExists(directory)
+}
+
 func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error {
 	for _, repo := range HelmRepositories {
 		helmAgent, err := NewHelmAgent("default", DeployLogPrinter)

plugins/scheduler-k3s/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/dokku/dokku/plugins/logs v0.0.0-20250618161309-8d0c35f1333c
 	github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-20250618161309-8d0c35f1333c
 	github.com/fatih/color v1.18.0
+	github.com/fluxcd/pkg/kustomize v1.18.0
 	github.com/go-openapi/jsonpointer v0.21.1
 	github.com/go-resty/resty/v2 v2.16.5
 	github.com/gofrs/flock v0.12.1
@@ -30,6 +31,7 @@ require (
 	k8s.io/kubernetes v1.33.2
 	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
 	mvdan.cc/sh/v3 v3.12.0
+	sigs.k8s.io/kustomize/api v0.20.0
 )
 
 require (
@@ -168,7 +170,6 @@ require (
 	sigs.k8s.io/controller-runtime v0.21.0 // indirect
 	sigs.k8s.io/gateway-api v1.3.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
-	sigs.k8s.io/kustomize/api v0.20.0 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.20.0 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect

plugins/scheduler-k3s/go.sum

@@ -89,6 +89,8 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fluxcd/pkg/kustomize v1.18.0 h1:wWK+qYwmBmba3N3VAqZ9ijnfVGGaIjcaHWo033URZTw=
+github.com/fluxcd/pkg/kustomize v1.18.0/go.mod h1:Ij9722MdWIE6B1EPg2ZJUf6npycgfRfN4Lohi7D/Kic=
 github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=
 github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=

plugins/scheduler-k3s/helm.go

@@ -14,6 +14,7 @@ import (
 	"unicode"
 
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/fluxcd/pkg/kustomize/filesys"
 	"github.com/gofrs/flock"
 	"gopkg.in/yaml.v3"
 	"helm.sh/helm/v3/pkg/action"
@@ -21,9 +22,13 @@ import (
 	"helm.sh/helm/v3/pkg/cli"
 	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/kube"
+	"helm.sh/helm/v3/pkg/postrender"
 	"helm.sh/helm/v3/pkg/release"
 	"helm.sh/helm/v3/pkg/repo"
 	"helm.sh/helm/v3/pkg/storage/driver"
+	"sigs.k8s.io/kustomize/api/konfig"
+	"sigs.k8s.io/kustomize/api/krusty"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 var DevNullPrinter = func(format string, v ...interface{}) {}
@@ -46,16 +51,37 @@ var DeployLogPrinter = func(format string, v ...interface{}) {
 	}
 }
 
+// ChartInput is the input for the InstallOrUpgradeChart function
 type ChartInput struct {
-	ChartPath         string
-	Namespace         string
-	ReleaseName       string
-	RepoURL           string
+	// ChartPath is the path to the chart to install or upgrade
+	ChartPath string
+
+	// KustomizeRootPath is the path to the kustomize root path to use
+	KustomizeRootPath string
+
+	// Namespace is the namespace to install or upgrade the chart in
+	Namespace string
+
+	// ReleaseName is the name of the release to install or upgrade
+	ReleaseName string
+
+	// RepoURL is the URL of the chart repository to install or upgrade the chart from
+	RepoURL string
+
+	// RollbackOnFailure is whether to rollback on failure
 	RollbackOnFailure bool
-	Timeout           time.Duration
-	Wait              bool
-	Version           string
-	Values            map[string]interface{}
+
+	// Timeout is the timeout for the install or upgrade
+	Timeout time.Duration
+
+	// Wait is whether to wait for the install or upgrade to complete
+	Wait bool
+
+	// Version is the version of the chart to install or upgrade
+	Version string
+
+	// Values is the values to pass to the chart
+	Values map[string]interface{}
 }
 
 type Release struct {
@@ -237,11 +263,22 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error {
 		input.Values = map[string]interface{}{}
 	}
 
+	kustomizeRenderer := KustomizeRenderer{
+		KustomizeRootPath: input.KustomizeRootPath,
+	}
+
 	client := action.NewInstall(h.Configuration)
 	client.Atomic = false
 	client.ChartPathOptions = action.ChartPathOptions{}
 	client.CreateNamespace = true
 	client.DryRun = false
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		client.PostRenderer = &DebugRenderer{
+			Renderer: &kustomizeRenderer,
+		}
+	} else {
+		client.PostRenderer = &kustomizeRenderer
+	}
 	client.Namespace = namespace
 	client.ReleaseName = input.ReleaseName
 	client.Timeout = input.Timeout
@@ -333,16 +370,6 @@ func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) {
 	return releases, nil
 }
 
-type DebugRenderer struct {
-}
-
-func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
-	for _, line := range strings.Split(renderedManifests.String(), "\n") {
-		common.LogWarn(line)
-	}
-	return renderedManifests, nil
-}
-
 func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
 	namespace := input.Namespace
 	if namespace == "" {
@@ -359,14 +386,21 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
 		input.Values = map[string]interface{}{}
 	}
 
+	kustomizeRenderer := KustomizeRenderer{
+		KustomizeRootPath: input.KustomizeRootPath,
+	}
+
 	client := action.NewUpgrade(h.Configuration)
 	client.Atomic = input.RollbackOnFailure
 	client.ChartPathOptions = action.ChartPathOptions{}
 	client.CleanupOnFail = true
 	client.MaxHistory = 10
 	if os.Getenv("DOKKU_TRACE") == "1" {
-		client.DryRun = true
-		client.PostRenderer = &DebugRenderer{}
+		client.PostRenderer = &DebugRenderer{
+			Renderer: &kustomizeRenderer,
+		}
+	} else {
+		client.PostRenderer = &kustomizeRenderer
 	}
 	client.Namespace = namespace
 	client.Timeout = input.Timeout
@@ -413,3 +447,71 @@ func (h *HelmAgent) UninstallChart(releaseName string) error {
 
 	return nil
 }
+
+type DebugRenderer struct {
+	Renderer postrender.PostRenderer
+}
+
+func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
+	renderedManifests, err := p.Renderer.Run(renderedManifests)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, line := range strings.Split(renderedManifests.String(), "\n") {
+		common.LogWarn(line)
+	}
+	return renderedManifests, nil
+}
+
+type KustomizeRenderer struct {
+	KustomizeRootPath string
+}
+
+func (p *KustomizeRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
+	if p.KustomizeRootPath == "" {
+		return renderedManifests, nil
+	}
+
+	if !common.DirectoryExists(p.KustomizeRootPath) {
+		return renderedManifests, nil
+	}
+
+	fs, err := filesys.MakeFsOnDiskSecureBuild(p.KustomizeRootPath)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating filesystem: %w", err)
+	}
+
+	var kfile string
+	for _, f := range konfig.RecognizedKustomizationFileNames() {
+		if kf := filepath.Join(p.KustomizeRootPath, f); fs.Exists(kf) {
+			kfile = kf
+			break
+		}
+	}
+	if kfile == "" {
+		return nil, fmt.Errorf("%s not found", konfig.DefaultKustomizationFileName())
+	}
+
+	if err := fs.WriteFile(filepath.Join(p.KustomizeRootPath, "rendered.yaml"), renderedManifests.Bytes()); err != nil {
+		return nil, fmt.Errorf("Error writing rendered.yaml: %w", err)
+	}
+
+	buildOptions := &krusty.Options{
+		LoadRestrictions: types.LoadRestrictionsNone,
+		PluginConfig:     types.DisabledPluginConfig(),
+	}
+
+	k := krusty.MakeKustomizer(buildOptions)
+	m, err := k.Run(fs, p.KustomizeRootPath)
+	if err != nil {
+		return nil, err
+	}
+
+	resources, err := m.AsYaml()
+	if err != nil {
+		return nil, err
+	}
+
+	return bytes.NewBuffer(resources), nil
+}

plugins/scheduler-k3s/k8s.go

@@ -416,6 +416,35 @@ func (k KubernetesClient) ExecCommand(ctx context.Context, input ExecCommandInpu
 	})
 }
 
+// GetPodLogsInput contains all the information needed to get the logs for a Kubernetes pod
+type GetLogsInput struct {
+	// Name is the Kubernetes pod name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// GetLogs gets the logs for a Kubernetes pod
+func (k KubernetesClient) GetLogs(ctx context.Context, input GetLogsInput) ([]byte, error) {
+	logOptions := corev1.PodLogOptions{}
+
+	request := k.Client.CoreV1().Pods(input.Namespace).GetLogs(input.Name, &logOptions)
+
+	readCloser, err := request.Stream(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get logs: %w", err)
+	}
+	defer readCloser.Close()
+
+	bytes, err := io.ReadAll(readCloser)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read logs: %w", err)
+	}
+
+	return bytes, nil
+}
+
 // GetNodeInput contains all the information needed to get a Kubernetes node
 type GetNodeInput struct {
 	// Name is the Kubernetes node name

plugins/scheduler-k3s/report.go

@@ -28,6 +28,9 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		"--scheduler-k3s-global-ingress-class":          reportGlobalIngressClass,
 		"--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd,
 		"--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag,
+		"--scheduler-k3s-computed-kustomize-root-path":  reportComputedKustomizeRootPath,
+		"--scheduler-k3s-kustomize-root-path":           reportKustomizeRootPath,
+		"--scheduler-k3s-global-kustomize-root-path":    reportGlobalKustomizeRootPath,
 		"--scheduler-k3s-computed-namespace":            reportComputedNamespace,
 		"--scheduler-k3s-namespace":                     reportNamespace,
 		"--scheduler-k3s-global-namespace":              reportGlobalNamespace,
@@ -148,6 +151,18 @@ func reportGlobalLetsencryptEmailStag(appName string) string {
 	return getGlobalLetsencryptEmailStag()
 }
 
+func reportComputedKustomizeRootPath(appName string) string {
+	return getComputedKustomizeRootPath(appName)
+}
+
+func reportKustomizeRootPath(appName string) string {
+	return getKustomizeRootPath(appName)
+}
+
+func reportGlobalKustomizeRootPath(appName string) string {
+	return getGlobalKustomizeRootPath()
+}
+
 func reportComputedNamespace(appName string) string {
 	return getComputedNamespace(appName)
 }

plugins/scheduler-k3s/scheduler_k3s.go

@@ -20,6 +20,7 @@ var (
 	DefaultProperties = map[string]string{
 		"deploy-timeout":      "",
 		"letsencrypt-server":  "",
+		"kustomize-root-path": "",
 		"image-pull-secrets":  "",
 		"namespace":           "",
 		"rollback-on-failure": "",
@@ -33,6 +34,7 @@ var (
 		"ingress-class":          true,
 		"kube-context":           true,
 		"kubeconfig-path":        true,
+		"kustomize-root-path":    true,
 		"letsencrypt-server":     true,
 		"letsencrypt-email-prod": true,
 		"letsencrypt-email-stag": true,

plugins/scheduler-k3s/src/triggers/triggers.go

@@ -22,6 +22,13 @@ func main() {
 
 	var err error
 	switch trigger {
+	case "core-post-deploy":
+		appName := flag.Arg(0)
+		err = scheduler_k3s.TriggerCorePostDeploy(appName)
+	case "core-post-extract":
+		appName := flag.Arg(0)
+		sourceWorkDir := flag.Arg(1)
+		err = scheduler_k3s.TriggerCorePostExtract(appName, sourceWorkDir)
 	case "install":
 		err = scheduler_k3s.TriggerInstall()
 	case "post-app-clone-setup":
@@ -32,6 +39,9 @@ func main() {
 		oldAppName := flag.Arg(0)
 		newAppName := flag.Arg(1)
 		err = scheduler_k3s.TriggerPostAppRenameSetup(oldAppName, newAppName)
+	case "post-create":
+		appName := flag.Arg(0)
+		err = scheduler_k3s.TriggerPostCreate(appName)
 	case "post-delete":
 		appName := flag.Arg(0)
 		err = scheduler_k3s.TriggerPostDelete(appName)

plugins/scheduler-k3s/templates/chart/certificate.yaml

@@ -1,6 +1,6 @@
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 

plugins/scheduler-k3s/templates/chart/deployment.yaml

@@ -1,11 +1,11 @@
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 
 {{- $mappings := dict }}
-{{- if eq $processName "web" }}
+{{- if hasKey $config "web" }}
 {{- range $pdx, $port_map := $config.web.port_maps }}
 {{- $mappings := set $mappings $port_map.name "true" }}
 {{- end }}
@@ -61,7 +61,7 @@ spec:
         {{- range $config.args }}
         - {{ . }}
         {{- end }}
-        {{- if eq $processName "web" }}
+        {{- if hasKey $config "web" }}
         env:
         - name: PORT
           value: "{{ $.Values.global.network.primary_port }}"
@@ -73,7 +73,7 @@ spec:
         image: {{ $.Values.global.image.name }}
         imagePullPolicy: Always
         name: {{ $.Values.global.app_name }}-{{ $processName }}
-        {{- if eq $processName "web" }}
+        {{- if hasKey $config "web" }}
         ports:
         {{- range $pdx, $port_map := $config.web.port_maps }}
         {{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }}

plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml

@@ -1,6 +1,6 @@
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 

plugins/scheduler-k3s/templates/chart/ingress-route.yaml

@@ -1,6 +1,6 @@
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 

plugins/scheduler-k3s/templates/chart/ingress.yaml

@@ -1,11 +1,11 @@
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 
 {{- $routeToKeda := false }}
-{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
+{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }}
 {{- if and ($.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }}
 {{- $routeToKeda = true }}
 {{- end }}

plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml

@@ -1,11 +1,11 @@
 {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}}
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 
-{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
+{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }}
 {{- if and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http") }}
 {{- $mappings := dict }}
 {{- range $pdx, $port_map := $config.web.port_maps }}

plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml

@@ -1,11 +1,11 @@
 {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}}
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 
-{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
+{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }}
 ---
 kind: Service
 apiVersion: v1

plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml

@@ -1,7 +1,7 @@
 {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}}
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 

plugins/scheduler-k3s/templates/chart/service.yaml

@@ -1,6 +1,6 @@
 {{- range $processName, $config := .Values.processes }}
-# Skip cron jobs
-{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }}
+{{- if not (hasKey $config "web") }}
+# Skip {{ $processName }} as it doesn't have a web section
 {{- continue }}
 {{- end }}
 

plugins/scheduler-k3s/triggers.go

@@ -32,11 +32,50 @@ import (
 	"k8s.io/kubernetes/pkg/client/conditions"
 )
 
+// TriggerCorePostDeploy moves a configured kustomize root path to be in the app root dir
+func TriggerCorePostDeploy(appName string) error {
+	return common.CorePostDeploy(common.CorePostDeployInput{
+		AppName:     appName,
+		Destination: common.GetAppDataDirectory("scheduler-k3s", appName),
+		PluginName:  "scheduler-k3s",
+		ExtractedPaths: []common.CorePostDeployPath{
+			{Path: "kustomization", IsDirectory: true},
+		},
+	})
+
+	return nil
+}
+
+// TriggerCorePostExtract moves a configured kustomize root path to be in the app root dir
+func TriggerCorePostExtract(appName string, sourceWorkDir string) error {
+	destination := common.GetAppDataDirectory("scheduler-k3s", appName)
+	kustomizeRootPath := getComputedKustomizeRootPath(appName)
+	return common.CorePostExtract(common.CorePostExtractInput{
+		AppName:       appName,
+		Destination:   destination,
+		PluginName:    "scheduler-k3s",
+		SourceWorkDir: sourceWorkDir,
+		ToExtract: []common.CorePostExtractToExtract{
+			{
+				Path:        kustomizeRootPath,
+				IsDirectory: true,
+				Name:        "config/kustomize",
+				Destination: "kustomization",
+			},
+		},
+	})
+}
+
 // TriggerInstall runs the install step for the scheduler-k3s plugin
 func TriggerInstall() error {
 	if err := common.PropertySetup("scheduler-k3s"); err != nil {
 		return fmt.Errorf("Unable to install the scheduler-k3s plugin: %s", err.Error())
 	}
+
+	if err := common.SetupAppData("scheduler-k3s"); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -63,6 +102,11 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
 	return nil
 }
 
+// TriggerPostCreate creates the scheduler-k3s data directory
+func TriggerPostCreate(appName string) error {
+	return common.CreateAppDataDirectory("scheduler-k3s", appName)
+}
+
 // TriggerPostDelete destroys the scheduler-k3s data for a given app container
 func TriggerPostDelete(appName string) error {
 	dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName)
@@ -704,9 +748,15 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		}
 	}
 
+	kustomizeRootPath := ""
+	if hasKustomizeDirectory(appName) {
+		kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName)
+	}
+
 	common.LogInfo2(fmt.Sprintf("Installing %s", appName))
 	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
 		ChartPath:         chartPath,
+		KustomizeRootPath: kustomizeRootPath,
 		Namespace:         namespace,
 		ReleaseName:       appName,
 		RollbackOnFailure: allowRollbacks,

tests/unit/scheduler-k3s-3.bats

@@ -0,0 +1,65 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown_() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s) kustomize" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP inject_kustomization_yaml
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.replicas'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "3"
+}
+
+inject_kustomization_yaml() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  mkdir -p "$APP_REPO_DIR/config/kustomize"
+  touch "$APP_REPO_DIR/config/kustomize/kustomization.yaml"
+  cat <<EOF >"$APP_REPO_DIR/config/kustomize/kustomization.yaml"
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- rendered.yaml
+patches:
+- patch: |-
+    - op: replace
+      path: /spec/replicas
+      value: 3
+  target:
+      group: apps
+      version: v1
+      kind: Deployment
+EOF
+}