From e034421ea9ce42d320133ae6a3bcfa8f80e13348 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 8 Jul 2025 00:42:57 -0400 Subject: [PATCH 01/28] fix: ensure the InstallChart function also attaches the debug renderer --- plugins/scheduler-k3s/helm.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 8f007cdf1..da4cd008c 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -242,6 +242,10 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { client.ChartPathOptions = action.ChartPathOptions{} client.CreateNamespace = true client.DryRun = false + if os.Getenv("DOKKU_TRACE") == "1" { + client.DryRun = true + client.PostRenderer = &DebugRenderer{} + } client.Namespace = namespace client.ReleaseName = input.ReleaseName client.Timeout = input.Timeout From 5ae6c1c84531c1d0774a0801a26e8f9d813bd074 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 8 Jul 2025 20:52:26 -0400 Subject: [PATCH 02/28] feat: add the ability to specify a kustomize-root-path This will default to `config/kustomize`, though may change to `.dokku/kustomize` in the future. --- plugins/scheduler-k3s/functions.go | 17 +++++++++++++++++ plugins/scheduler-k3s/report.go | 15 +++++++++++++++ plugins/scheduler-k3s/scheduler_k3s.go | 2 ++ 3 files changed, 34 insertions(+) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 942d5c4f2..a7ec92f92 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1190,6 +1190,23 @@ func getGlobalLetsencryptEmailStag() string { return common.PropertyGetDefault("scheduler-k3s", "--global", "letsencrypt-email-stag", "") } +func getKustomizeRootPath(appName string) string { + return common.PropertyGetDefault("scheduler-k3s", appName, "kustomize-root-path", "") +} + +func getGlobalKustomizeRootPath() string { + return common.PropertyGetDefault("scheduler-k3s", "--global", "kustomize-root-path", "config/kustomize") +} + +func getComputedKustomizeRootPath(appName string) string { + kustomizeRootPath := getKustomizeRootPath(appName) + if kustomizeRootPath == "" { + kustomizeRootPath = getGlobalKustomizeRootPath() + } + + return kustomizeRootPath +} + func getNamespace(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "") } diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index d480969b6..d1bb6d2de 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -28,6 +28,9 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-global-ingress-class": reportGlobalIngressClass, "--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd, "--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag, + "--scheduler-k3s-computed-kustomize-root-path": reportComputedKustomizeRootPath, + "--scheduler-k3s-kustomize-root-path": reportKustomizeRootPath, + "--scheduler-k3s-global-kustomize-root-path": reportGlobalKustomizeRootPath, "--scheduler-k3s-computed-namespace": reportComputedNamespace, "--scheduler-k3s-namespace": reportNamespace, "--scheduler-k3s-global-namespace": reportGlobalNamespace, @@ -148,6 +151,18 @@ func reportGlobalLetsencryptEmailStag(appName string) string { return getGlobalLetsencryptEmailStag() } +func reportComputedKustomizeRootPath(appName string) string { + return getComputedKustomizeRootPath(appName) +} + +func reportKustomizeRootPath(appName string) string { + return getKustomizeRootPath(appName) +} + +func reportGlobalKustomizeRootPath(appName string) string { + return getGlobalKustomizeRootPath() +} + func reportComputedNamespace(appName string) string { return getComputedNamespace(appName) } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 9a37a78bd..fbf29a7bc 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -20,6 +20,7 @@ var ( DefaultProperties = map[string]string{ "deploy-timeout": "", "letsencrypt-server": "", + "kustomize-root-path": "", "image-pull-secrets": "", "namespace": "", "rollback-on-failure": "", @@ -33,6 +34,7 @@ var ( "ingress-class": true, "kube-context": true, "kubeconfig-path": true, + "kustomize-root-path": true, "letsencrypt-server": true, "letsencrypt-email-prod": true, "letsencrypt-email-stag": true, From 1b4e02eb6234eb3492a944e5d9647543f164bcfe Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 8 Jul 2025 22:30:20 -0400 Subject: [PATCH 03/28] feat: create a KustomizeRenderer The KustomizeRenderer is used to invoke Kustomize to the generated renderer.yaml. --- plugins/scheduler-k3s/go.mod | 3 +- plugins/scheduler-k3s/go.sum | 2 + plugins/scheduler-k3s/helm.go | 89 +++++++++++++++++++++++++++++++---- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index 2e7f2df80..bf48e9f6f 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -11,6 +11,7 @@ require ( github.com/dokku/dokku/plugins/logs v0.0.0-20250618161309-8d0c35f1333c github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-20250618161309-8d0c35f1333c github.com/fatih/color v1.18.0 + github.com/fluxcd/pkg/kustomize v1.18.0 github.com/go-openapi/jsonpointer v0.21.1 github.com/go-resty/resty/v2 v2.16.5 github.com/gofrs/flock v0.12.1 @@ -30,6 +31,7 @@ require ( k8s.io/kubernetes v1.33.2 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 mvdan.cc/sh/v3 v3.12.0 + sigs.k8s.io/kustomize/api v0.20.0 ) require ( @@ -168,7 +170,6 @@ require ( sigs.k8s.io/controller-runtime v0.21.0 // indirect sigs.k8s.io/gateway-api v1.3.0 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect - sigs.k8s.io/kustomize/api v0.20.0 // indirect sigs.k8s.io/kustomize/kyaml v0.20.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum index 46f0a044d..08511dbb7 100644 --- a/plugins/scheduler-k3s/go.sum +++ b/plugins/scheduler-k3s/go.sum @@ -89,6 +89,8 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fluxcd/pkg/kustomize v1.18.0 h1:wWK+qYwmBmba3N3VAqZ9ijnfVGGaIjcaHWo033URZTw= +github.com/fluxcd/pkg/kustomize v1.18.0/go.mod h1:Ij9722MdWIE6B1EPg2ZJUf6npycgfRfN4Lohi7D/Kic= github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index da4cd008c..8eb42d4a8 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -14,6 +14,7 @@ import ( "unicode" "github.com/dokku/dokku/plugins/common" + "github.com/fluxcd/pkg/kustomize/filesys" "github.com/gofrs/flock" "gopkg.in/yaml.v3" "helm.sh/helm/v3/pkg/action" @@ -21,9 +22,13 @@ import ( "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/kube" + "helm.sh/helm/v3/pkg/postrender" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/storage/driver" + "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/api/types" ) var DevNullPrinter = func(format string, v ...interface{}) {} @@ -337,16 +342,6 @@ func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) { return releases, nil } -type DebugRenderer struct { -} - -func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { - for _, line := range strings.Split(renderedManifests.String(), "\n") { - common.LogWarn(line) - } - return renderedManifests, nil -} - func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { namespace := input.Namespace if namespace == "" { @@ -417,3 +412,77 @@ func (h *HelmAgent) UninstallChart(releaseName string) error { return nil } + +type DebugRenderer struct { + Renderer *postrender.PostRenderer +} + +func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { + if p.Renderer != nil { + var err error + renderer := *p.Renderer + renderedManifests, err = renderer.Run(renderedManifests) + if err != nil { + return nil, err + } + } + + for _, line := range strings.Split(renderedManifests.String(), "\n") { + common.LogWarn(line) + } + return renderedManifests, nil +} + +type KustomizeRenderer struct { + Renderer *postrender.PostRenderer + KustomizeRootPath string +} + +func (p *KustomizeRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { + if p.KustomizeRootPath == "" { + return nil, nil + } + + if !common.DirectoryExists(p.KustomizeRootPath) { + return nil, nil + } + + fs, err := filesys.MakeFsOnDiskSecureBuild(p.KustomizeRootPath) + if err != nil { + return nil, fmt.Errorf("Error creating filesystem: %w", err) + } + + var kfile string + for _, f := range konfig.RecognizedKustomizationFileNames() { + if kf := filepath.Join(p.KustomizeRootPath, f); fs.Exists(kf) { + kfile = kf + break + } + } + if kfile == "" { + return nil, fmt.Errorf("%s not found", konfig.DefaultKustomizationFileName()) + } + + renderedYamlPath := filepath.Join(p.KustomizeRootPath, "rendered.yaml") + if err := os.WriteFile(renderedYamlPath, renderedManifests.Bytes(), 0644); err != nil { + return nil, fmt.Errorf("Error writing rendered.yaml: %w", err) + } + + buildOptions := &krusty.Options{ + LoadRestrictions: types.LoadRestrictionsNone, + PluginConfig: types.DisabledPluginConfig(), + } + + k := krusty.MakeKustomizer(buildOptions) + m, err := k.Run(fs, p.KustomizeRootPath) + if err != nil { + return nil, err + } + + resources, err := m.AsYaml() + if err != nil { + return nil, err + } + + return bytes.NewBuffer(resources), nil +} From e0617b209d0c93feb3464f5aee810df258c74a10 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 19 Jul 2025 00:05:44 -0400 Subject: [PATCH 04/28] feat: extract kustomize directory from repo during deploys --- plugins/scheduler-k3s/Makefile | 2 +- plugins/scheduler-k3s/functions.go | 13 +++ .../scheduler-k3s/src/triggers/triggers.go | 7 ++ plugins/scheduler-k3s/triggers.go | 80 +++++++++++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index 449103c2c..c0c9165bd 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall -TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop +TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index a7ec92f92..472c06025 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1494,6 +1494,19 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { }, nil } +func hasKustomizeRootPath(appName string) bool { + kustomizeRootPath := getComputedKustomizeRootPath(appName) + if common.DirectoryExists(fmt.Sprintf("%s.%s.missing", kustomizeRootPath, os.Getenv("DOKKU_PID"))) { + return false + } + + if common.DirectoryExists(fmt.Sprintf("%s.%s", kustomizeRootPath, os.Getenv("DOKKU_PID"))) { + return true + } + + return common.DirectoryExists(kustomizeRootPath) +} + func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error { for _, repo := range HelmRepositories { helmAgent, err := NewHelmAgent("default", DeployLogPrinter) diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go index 4092da096..27b6bdc98 100644 --- a/plugins/scheduler-k3s/src/triggers/triggers.go +++ b/plugins/scheduler-k3s/src/triggers/triggers.go @@ -22,6 +22,13 @@ func main() { var err error switch trigger { + case "core-post-deploy": + appName := flag.Arg(0) + err = scheduler_k3s.TriggerCorePostDeploy(appName) + case "core-post-extract": + appName := flag.Arg(0) + sourceWorkDir := flag.Arg(1) + err = scheduler_k3s.TriggerCorePostExtract(appName, sourceWorkDir) case "install": err = scheduler_k3s.TriggerInstall() case "post-app-clone-setup": diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index a228ac048..5fb53b6f1 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -11,6 +11,7 @@ import ( "io" "os" "os/signal" + "path" "path/filepath" "sort" "strconv" @@ -32,6 +33,85 @@ import ( "k8s.io/kubernetes/pkg/client/conditions" ) +// TriggerCorePostDeploy moves a configured kustomize root path to be in the app root dir +func TriggerCorePostDeploy(appName string) error { + existingKustomizeRootPath := getComputedKustomizeRootPath(appName) + processSpecificKustomizeRootPath := fmt.Sprintf("%s.%s", existingKustomizeRootPath, os.Getenv("DOKKU_PID")) + if common.DirectoryExists(processSpecificKustomizeRootPath) { + if err := os.Rename(processSpecificKustomizeRootPath, existingKustomizeRootPath); err != nil { + return err + } + } else if common.FileExists(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)) { + if err := os.Remove(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)); err != nil { + return err + } + + if common.DirectoryExists(existingKustomizeRootPath) { + if err := os.Remove(existingKustomizeRootPath); err != nil { + return err + } + } + } + + return nil +} + +// TriggerCorePostExtract moves a configured kustomize root path to be in the app root dir +func TriggerCorePostExtract(appName string, sourceWorkDir string) error { + kustomizeRootPath := getComputedKustomizeRootPath(appName) + if kustomizeRootPath == "" { + return nil + } + + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName) + existingKustomizeDirectory := filepath.Join(directory, "kustomization") + files, err := filepath.Glob(fmt.Sprintf("%s.*", existingKustomizeDirectory)) + if err != nil { + return err + } + for _, f := range files { + if err := os.Remove(f); err != nil { + return err + } + } + + processSpecificKustomizeRootPath := fmt.Sprintf("%s.%s", kustomizeRootPath, os.Getenv("DOKKU_PID")) + results, _ := common.CallPlugnTrigger(common.PlugnTriggerInput{ + Trigger: "git-get-property", + Args: []string{appName, "source-image"}, + }) + appSourceImage := results.StdoutContents() + + if appSourceImage == "" { + repoDefaultKustomizeRootPath := path.Join(sourceWorkDir, "kustomization") + repoKustomizeRootPath := path.Join(sourceWorkDir, kustomizeRootPath) + if !common.DirectoryExists(repoKustomizeRootPath) { + if kustomizeRootPath != "kustomization" && common.DirectoryExists(repoDefaultKustomizeRootPath) { + if err := os.RemoveAll(repoDefaultKustomizeRootPath); err != nil { + return fmt.Errorf("Unable to remove existing kustomize directory: %s", err.Error()) + } + } + return common.TouchDir(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)) + } + + if err := common.Copy(repoKustomizeRootPath, processSpecificKustomizeRootPath); err != nil { + return fmt.Errorf("Unable to extract kustomize root path: %s", err.Error()) + } + + if kustomizeRootPath != "kustomization" { + if err := common.Copy(repoKustomizeRootPath, repoDefaultKustomizeRootPath); err != nil { + return fmt.Errorf("Unable to move kustomize root path into place: %s", err.Error()) + } + } + } else { + if err := common.CopyDirFromImage(appName, appSourceImage, kustomizeRootPath, processSpecificKustomizeRootPath); err != nil { + return common.TouchDir(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)) + } + } + + return nil +} + // TriggerInstall runs the install step for the scheduler-k3s plugin func TriggerInstall() error { if err := common.PropertySetup("scheduler-k3s"); err != nil { From 04917b72d43335802d262d7fa0e24ad99b5a26f9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 19 Jul 2025 00:13:38 -0400 Subject: [PATCH 05/28] fix: ensure the directories are deleted --- plugins/scheduler-k3s/triggers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 5fb53b6f1..4f3366c51 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -42,12 +42,12 @@ func TriggerCorePostDeploy(appName string) error { return err } } else if common.FileExists(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)) { - if err := os.Remove(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)); err != nil { + if err := os.RemoveAll(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)); err != nil { return err } if common.DirectoryExists(existingKustomizeRootPath) { - if err := os.Remove(existingKustomizeRootPath); err != nil { + if err := os.RemoveAll(existingKustomizeRootPath); err != nil { return err } } From e83ba595aadf71ffa47e5994027e7e918bed6916 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 19 Jul 2025 00:37:46 -0400 Subject: [PATCH 06/28] fix: correct issues with passing a KustomizeRenderer --- plugins/scheduler-k3s/helm.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 8eb42d4a8..0a0310332 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -414,17 +414,13 @@ func (h *HelmAgent) UninstallChart(releaseName string) error { } type DebugRenderer struct { - Renderer *postrender.PostRenderer + Renderer postrender.PostRenderer } func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { - if p.Renderer != nil { - var err error - renderer := *p.Renderer - renderedManifests, err = renderer.Run(renderedManifests) - if err != nil { - return nil, err - } + renderedManifests, err := p.Renderer.Run(renderedManifests) + if err != nil { + return nil, err } for _, line := range strings.Split(renderedManifests.String(), "\n") { @@ -434,7 +430,6 @@ func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, err } type KustomizeRenderer struct { - Renderer *postrender.PostRenderer KustomizeRootPath string } From 272327af89c91888837c10cf03bea0fc1f11fc35 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 19 Jul 2025 00:38:06 -0400 Subject: [PATCH 07/28] chore: add docs to the ChartInput type --- plugins/scheduler-k3s/helm.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 0a0310332..f40502778 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -51,16 +51,34 @@ var DeployLogPrinter = func(format string, v ...interface{}) { } } +// ChartInput is the input for the InstallOrUpgradeChart function type ChartInput struct { - ChartPath string - Namespace string - ReleaseName string - RepoURL string + // ChartPath is the path to the chart to install or upgrade + ChartPath string + + // Namespace is the namespace to install or upgrade the chart in + Namespace string + + // ReleaseName is the name of the release to install or upgrade + ReleaseName string + + // RepoURL is the URL of the chart repository to install or upgrade the chart from + RepoURL string + + // RollbackOnFailure is whether to rollback on failure RollbackOnFailure bool - Timeout time.Duration - Wait bool - Version string - Values map[string]interface{} + + // Timeout is the timeout for the install or upgrade + Timeout time.Duration + + // Wait is whether to wait for the install or upgrade to complete + Wait bool + + // Version is the version of the chart to install or upgrade + Version string + + // Values is the values to pass to the chart + Values map[string]interface{} } type Release struct { From d0ffd7d9f97a9d50156613df07157cef55910eb4 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 00:04:35 -0400 Subject: [PATCH 08/28] feat: stub out kustomize renderer The process-specific kustomize root path is not currently set, but will be soon. --- plugins/scheduler-k3s/functions.go | 4 ++++ plugins/scheduler-k3s/helm.go | 23 +++++++++++++++++++++-- plugins/scheduler-k3s/triggers.go | 6 ++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 472c06025..2fe549f6e 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1494,6 +1494,10 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { }, nil } +func getProcessSpecificKustomizeRootPath(appName string) string { + return "" +} + func hasKustomizeRootPath(appName string) bool { kustomizeRootPath := getComputedKustomizeRootPath(appName) if common.DirectoryExists(fmt.Sprintf("%s.%s.missing", kustomizeRootPath, os.Getenv("DOKKU_PID"))) { diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index f40502778..4f855c13f 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -56,6 +56,9 @@ type ChartInput struct { // ChartPath is the path to the chart to install or upgrade ChartPath string + // KustomizeRootPath is the path to the kustomize root path to use + KustomizeRootPath string + // Namespace is the namespace to install or upgrade the chart in Namespace string @@ -260,6 +263,10 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { input.Values = map[string]interface{}{} } + kustomizeRenderer := KustomizeRenderer{ + KustomizeRootPath: input.KustomizeRootPath, + } + client := action.NewInstall(h.Configuration) client.Atomic = false client.ChartPathOptions = action.ChartPathOptions{} @@ -267,7 +274,11 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { client.DryRun = false if os.Getenv("DOKKU_TRACE") == "1" { client.DryRun = true - client.PostRenderer = &DebugRenderer{} + client.PostRenderer = &DebugRenderer{ + Renderer: &kustomizeRenderer, + } + } else { + client.PostRenderer = &kustomizeRenderer } client.Namespace = namespace client.ReleaseName = input.ReleaseName @@ -376,6 +387,10 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { input.Values = map[string]interface{}{} } + kustomizeRenderer := KustomizeRenderer{ + KustomizeRootPath: input.KustomizeRootPath, + } + client := action.NewUpgrade(h.Configuration) client.Atomic = input.RollbackOnFailure client.ChartPathOptions = action.ChartPathOptions{} @@ -383,7 +398,11 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { client.MaxHistory = 10 if os.Getenv("DOKKU_TRACE") == "1" { client.DryRun = true - client.PostRenderer = &DebugRenderer{} + client.PostRenderer = &DebugRenderer{ + Renderer: &kustomizeRenderer, + } + } else { + client.PostRenderer = &kustomizeRenderer } client.Namespace = namespace client.Timeout = input.Timeout diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 4f3366c51..c1b08d905 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -784,9 +784,15 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } } + kustomizeRootPath := "" + if hasKustomizeRootPath(appName) { + kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName) + } + common.LogInfo2(fmt.Sprintf("Installing %s", appName)) err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{ ChartPath: chartPath, + KustomizeRootPath: kustomizeRootPath, Namespace: namespace, ReleaseName: appName, RollbackOnFailure: allowRollbacks, From fd6e56cc5fbac8c6cbee00e5c9422d0e8ca7f0bc Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 00:11:25 -0400 Subject: [PATCH 09/28] fix: correct kustomize root path check --- plugins/scheduler-k3s/functions.go | 16 +++++++++++----- plugins/scheduler-k3s/triggers.go | 7 +++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 2fe549f6e..76f5991d7 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1190,6 +1190,11 @@ func getGlobalLetsencryptEmailStag() string { return common.PropertyGetDefault("scheduler-k3s", "--global", "letsencrypt-email-stag", "") } +func getKustomizeDirectory(appName string) string { + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName) + return filepath.Join(directory, "kustomization") +} + func getKustomizeRootPath(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "kustomize-root-path", "") } @@ -1498,17 +1503,18 @@ func getProcessSpecificKustomizeRootPath(appName string) string { return "" } -func hasKustomizeRootPath(appName string) bool { - kustomizeRootPath := getComputedKustomizeRootPath(appName) - if common.DirectoryExists(fmt.Sprintf("%s.%s.missing", kustomizeRootPath, os.Getenv("DOKKU_PID"))) { +func hasKustomizeDirectory(appName string) bool { + directory := getKustomizeDirectory(appName) + + if common.DirectoryExists(fmt.Sprintf("%s.%s.missing", directory, os.Getenv("DOKKU_PID"))) { return false } - if common.DirectoryExists(fmt.Sprintf("%s.%s", kustomizeRootPath, os.Getenv("DOKKU_PID"))) { + if common.DirectoryExists(fmt.Sprintf("%s.%s", directory, os.Getenv("DOKKU_PID"))) { return true } - return common.DirectoryExists(kustomizeRootPath) + return common.DirectoryExists(directory) } func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error { diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index c1b08d905..f9fcd7817 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -63,8 +63,7 @@ func TriggerCorePostExtract(appName string, sourceWorkDir string) error { return nil } - directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName) - existingKustomizeDirectory := filepath.Join(directory, "kustomization") + existingKustomizeDirectory := getKustomizeDirectory(appName) files, err := filepath.Glob(fmt.Sprintf("%s.*", existingKustomizeDirectory)) if err != nil { return err @@ -75,7 +74,7 @@ func TriggerCorePostExtract(appName string, sourceWorkDir string) error { } } - processSpecificKustomizeRootPath := fmt.Sprintf("%s.%s", kustomizeRootPath, os.Getenv("DOKKU_PID")) + processSpecificKustomizeRootPath := fmt.Sprintf("%s.%s", existingKustomizeDirectory, os.Getenv("DOKKU_PID")) results, _ := common.CallPlugnTrigger(common.PlugnTriggerInput{ Trigger: "git-get-property", Args: []string{appName, "source-image"}, @@ -785,7 +784,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } kustomizeRootPath := "" - if hasKustomizeRootPath(appName) { + if hasKustomizeDirectory(appName) { kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName) } From 58c6b137b040157d7e9a123546ca8712c6369e64 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 01:48:15 -0400 Subject: [PATCH 10/28] refactor: use a common implementation to extract kustomize files --- plugins/scheduler-k3s/triggers.go | 58 +++++-------------------------- 1 file changed, 8 insertions(+), 50 deletions(-) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index f9fcd7817..d2f4f06c6 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -11,7 +11,6 @@ import ( "io" "os" "os/signal" - "path" "path/filepath" "sort" "strconv" @@ -58,57 +57,16 @@ func TriggerCorePostDeploy(appName string) error { // TriggerCorePostExtract moves a configured kustomize root path to be in the app root dir func TriggerCorePostExtract(appName string, sourceWorkDir string) error { + destination := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName) kustomizeRootPath := getComputedKustomizeRootPath(appName) - if kustomizeRootPath == "" { - return nil - } - - existingKustomizeDirectory := getKustomizeDirectory(appName) - files, err := filepath.Glob(fmt.Sprintf("%s.*", existingKustomizeDirectory)) - if err != nil { - return err - } - for _, f := range files { - if err := os.Remove(f); err != nil { - return err - } - } - - processSpecificKustomizeRootPath := fmt.Sprintf("%s.%s", existingKustomizeDirectory, os.Getenv("DOKKU_PID")) - results, _ := common.CallPlugnTrigger(common.PlugnTriggerInput{ - Trigger: "git-get-property", - Args: []string{appName, "source-image"}, + return common.CorePostExtract(common.CorePostExtractInput{ + AppName: appName, + SourceWorkDir: sourceWorkDir, + Destination: destination, + ToExtract: []common.CorePostExtractToExtract{ + {Path: kustomizeRootPath, IsDirectory: true, Destination: "kustomization"}, + }, }) - appSourceImage := results.StdoutContents() - - if appSourceImage == "" { - repoDefaultKustomizeRootPath := path.Join(sourceWorkDir, "kustomization") - repoKustomizeRootPath := path.Join(sourceWorkDir, kustomizeRootPath) - if !common.DirectoryExists(repoKustomizeRootPath) { - if kustomizeRootPath != "kustomization" && common.DirectoryExists(repoDefaultKustomizeRootPath) { - if err := os.RemoveAll(repoDefaultKustomizeRootPath); err != nil { - return fmt.Errorf("Unable to remove existing kustomize directory: %s", err.Error()) - } - } - return common.TouchDir(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)) - } - - if err := common.Copy(repoKustomizeRootPath, processSpecificKustomizeRootPath); err != nil { - return fmt.Errorf("Unable to extract kustomize root path: %s", err.Error()) - } - - if kustomizeRootPath != "kustomization" { - if err := common.Copy(repoKustomizeRootPath, repoDefaultKustomizeRootPath); err != nil { - return fmt.Errorf("Unable to move kustomize root path into place: %s", err.Error()) - } - } - } else { - if err := common.CopyDirFromImage(appName, appSourceImage, kustomizeRootPath, processSpecificKustomizeRootPath); err != nil { - return common.TouchDir(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)) - } - } - - return nil } // TriggerInstall runs the install step for the scheduler-k3s plugin From b24035c80acf4a8ddcd43c1a204afd0b7f69a98c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 01:58:44 -0400 Subject: [PATCH 11/28] refactor: use a common implementation to move files into place in golang core-post-deploy implementations --- plugins/scheduler-k3s/triggers.go | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index d2f4f06c6..a1a8edd70 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -34,30 +34,20 @@ import ( // TriggerCorePostDeploy moves a configured kustomize root path to be in the app root dir func TriggerCorePostDeploy(appName string) error { - existingKustomizeRootPath := getComputedKustomizeRootPath(appName) - processSpecificKustomizeRootPath := fmt.Sprintf("%s.%s", existingKustomizeRootPath, os.Getenv("DOKKU_PID")) - if common.DirectoryExists(processSpecificKustomizeRootPath) { - if err := os.Rename(processSpecificKustomizeRootPath, existingKustomizeRootPath); err != nil { - return err - } - } else if common.FileExists(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)) { - if err := os.RemoveAll(fmt.Sprintf("%s.missing", processSpecificKustomizeRootPath)); err != nil { - return err - } - - if common.DirectoryExists(existingKustomizeRootPath) { - if err := os.RemoveAll(existingKustomizeRootPath); err != nil { - return err - } - } - } + return common.CorePostDeploy(common.CorePostDeployInput{ + AppName: appName, + Destination: common.GetAppDataDirectory("scheduler-k3s", appName), + ExtractedPaths: []common.CorePostDeployPath{ + {Name: "kustomization", IsDirectory: true}, + }, + }) return nil } // TriggerCorePostExtract moves a configured kustomize root path to be in the app root dir func TriggerCorePostExtract(appName string, sourceWorkDir string) error { - destination := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName) + destination := common.GetAppDataDirectory("scheduler-k3s", appName) kustomizeRootPath := getComputedKustomizeRootPath(appName) return common.CorePostExtract(common.CorePostExtractInput{ AppName: appName, From b6fdd4511caf793d6313bec44ae8f5d13ec1ae2d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 02:23:41 -0400 Subject: [PATCH 12/28] fix: add validation to input for common trigger functions --- plugins/scheduler-k3s/triggers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index a1a8edd70..aa1e4f0ae 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -37,6 +37,7 @@ func TriggerCorePostDeploy(appName string) error { return common.CorePostDeploy(common.CorePostDeployInput{ AppName: appName, Destination: common.GetAppDataDirectory("scheduler-k3s", appName), + PluginName: "scheduler-k3s", ExtractedPaths: []common.CorePostDeployPath{ {Name: "kustomization", IsDirectory: true}, }, @@ -51,8 +52,9 @@ func TriggerCorePostExtract(appName string, sourceWorkDir string) error { kustomizeRootPath := getComputedKustomizeRootPath(appName) return common.CorePostExtract(common.CorePostExtractInput{ AppName: appName, - SourceWorkDir: sourceWorkDir, Destination: destination, + PluginName: "scheduler-k3s", + SourceWorkDir: sourceWorkDir, ToExtract: []common.CorePostExtractToExtract{ {Path: kustomizeRootPath, IsDirectory: true, Destination: "kustomization"}, }, From 563365b712d11ce43d857f66116e97bf26f18132 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 02:40:41 -0400 Subject: [PATCH 13/28] fix: add name to app-json trigger code and some formatting --- plugins/scheduler-k3s/triggers.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index aa1e4f0ae..d26056b99 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -39,7 +39,7 @@ func TriggerCorePostDeploy(appName string) error { Destination: common.GetAppDataDirectory("scheduler-k3s", appName), PluginName: "scheduler-k3s", ExtractedPaths: []common.CorePostDeployPath{ - {Name: "kustomization", IsDirectory: true}, + {Path: "kustomization", IsDirectory: true}, }, }) @@ -56,7 +56,12 @@ func TriggerCorePostExtract(appName string, sourceWorkDir string) error { PluginName: "scheduler-k3s", SourceWorkDir: sourceWorkDir, ToExtract: []common.CorePostExtractToExtract{ - {Path: kustomizeRootPath, IsDirectory: true, Destination: "kustomization"}, + { + Path: kustomizeRootPath, + IsDirectory: true, + Name: "config/kustomize", + Destination: "kustomization", + }, }, }) } From 0a86ffe01bde4343fe9a677cd20321165e95d486 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 03:14:51 -0400 Subject: [PATCH 14/28] fix: add missing scheduler-k3s directory --- plugins/scheduler-k3s/triggers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index d26056b99..5c28d3c22 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -71,6 +71,11 @@ func TriggerInstall() error { if err := common.PropertySetup("scheduler-k3s"); err != nil { return fmt.Errorf("Unable to install the scheduler-k3s plugin: %s", err.Error()) } + + if err := common.SetupAppData("scheduler-k3s"); err != nil { + return err + } + return nil } From 3edddac0c16a0536df2b24c83544204e7ffdd247 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 03:28:16 -0400 Subject: [PATCH 15/28] fix: create the data directory --- plugins/scheduler-k3s/Makefile | 2 +- plugins/scheduler-k3s/src/triggers/triggers.go | 3 +++ plugins/scheduler-k3s/triggers.go | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index c0c9165bd..870e3ba05 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall -TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop +TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go index 27b6bdc98..692c62e1f 100644 --- a/plugins/scheduler-k3s/src/triggers/triggers.go +++ b/plugins/scheduler-k3s/src/triggers/triggers.go @@ -39,6 +39,9 @@ func main() { oldAppName := flag.Arg(0) newAppName := flag.Arg(1) err = scheduler_k3s.TriggerPostAppRenameSetup(oldAppName, newAppName) + case "post-create": + appName := flag.Arg(0) + err = scheduler_k3s.TriggerPostCreate(appName) case "post-delete": appName := flag.Arg(0) err = scheduler_k3s.TriggerPostDelete(appName) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 5c28d3c22..7c0a9f8c4 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -102,6 +102,11 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error { return nil } +// TriggerPostCreate creates the scheduler-k3s data directory +func TriggerPostCreate(appName string) error { + return common.CreateAppDataDirectory("scheduler-k3s", appName) +} + // TriggerPostDelete destroys the scheduler-k3s data for a given app container func TriggerPostDelete(appName string) error { dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName) From 7595351b1d2d9126095e016f5f3e1d628e6aa57f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 15:42:41 -0400 Subject: [PATCH 16/28] debug: check on what pods are running in the cert-manager namespace --- tests/unit/scheduler-k3s-2.bats | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/scheduler-k3s-2.bats b/tests/unit/scheduler-k3s-2.bats index 60734ad07..4eec4820a 100644 --- a/tests/unit/scheduler-k3s-2.bats +++ b/tests/unit/scheduler-k3s-2.bats @@ -12,6 +12,7 @@ setup() { } teardown_() { + kubectl get pods -n cert-manager || true global_teardown dokku nginx:start uninstall_k3s || true From d33393d3fd7a456f9d73c9be11fb91424f78179f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 16:32:21 -0400 Subject: [PATCH 17/28] feat: debug failed global helm chart installations by checking all pod logs --- plugins/scheduler-k3s/functions.go | 19 +++++++++++++++++++ plugins/scheduler-k3s/k8s.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 76f5991d7..accb49d17 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1605,6 +1605,25 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn Wait: true, }) if err != nil { + chartPods, err := clientset.ListPods(ctx, ListPodsInput{ + Namespace: chart.Namespace, + }) + if err != nil { + return fmt.Errorf("Error listing chart pods: %w", err) + } + for _, pod := range chartPods { + logs, err := clientset.GetLogs(ctx, GetLogsInput{ + Name: pod.Name, + Namespace: pod.Namespace, + }) + if err != nil { + return fmt.Errorf("Error getting chart pod logs: %w", err) + } + common.LogExclaim(fmt.Sprintf("Logs for %s/%s", pod.Namespace, pod.Name)) + for _, log := range strings.Split(string(logs), "\n") { + common.LogInfo1(log) + } + } return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err) } } diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index 494d5ab9a..491e9174a 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -416,6 +416,35 @@ func (k KubernetesClient) ExecCommand(ctx context.Context, input ExecCommandInpu }) } +// GetPodLogsInput contains all the information needed to get the logs for a Kubernetes pod +type GetLogsInput struct { + // Name is the Kubernetes pod name + Name string + + // Namespace is the Kubernetes namespace + Namespace string +} + +// GetLogs gets the logs for a Kubernetes pod +func (k KubernetesClient) GetLogs(ctx context.Context, input GetLogsInput) ([]byte, error) { + logOptions := corev1.PodLogOptions{} + + request := k.Client.CoreV1().Pods(input.Namespace).GetLogs(input.Name, &logOptions) + + readCloser, err := request.Stream(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get logs: %w", err) + } + defer readCloser.Close() + + bytes, err := io.ReadAll(readCloser) + if err != nil { + return nil, fmt.Errorf("failed to read logs: %w", err) + } + + return bytes, nil +} + // GetNodeInput contains all the information needed to get a Kubernetes node type GetNodeInput struct { // Name is the Kubernetes node name From 8f44a571581bd79bac3256335b46c2f4452e911c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 17:28:36 -0400 Subject: [PATCH 18/28] fix: return the initial manifests if there is no kustomize root path --- plugins/scheduler-k3s/helm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 4f855c13f..98fedf630 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -472,11 +472,11 @@ type KustomizeRenderer struct { func (p *KustomizeRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { if p.KustomizeRootPath == "" { - return nil, nil + return renderedManifests, nil } if !common.DirectoryExists(p.KustomizeRootPath) { - return nil, nil + return renderedManifests, nil } fs, err := filesys.MakeFsOnDiskSecureBuild(p.KustomizeRootPath) From e7964ee99dc3d878856212003567383e61c8880c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 17:51:54 -0400 Subject: [PATCH 19/28] chore: remove debug code --- plugins/scheduler-k3s/functions.go | 19 ------------------- tests/unit/scheduler-k3s-2.bats | 1 - 2 files changed, 20 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index accb49d17..76f5991d7 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1605,25 +1605,6 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn Wait: true, }) if err != nil { - chartPods, err := clientset.ListPods(ctx, ListPodsInput{ - Namespace: chart.Namespace, - }) - if err != nil { - return fmt.Errorf("Error listing chart pods: %w", err) - } - for _, pod := range chartPods { - logs, err := clientset.GetLogs(ctx, GetLogsInput{ - Name: pod.Name, - Namespace: pod.Namespace, - }) - if err != nil { - return fmt.Errorf("Error getting chart pod logs: %w", err) - } - common.LogExclaim(fmt.Sprintf("Logs for %s/%s", pod.Namespace, pod.Name)) - for _, log := range strings.Split(string(logs), "\n") { - common.LogInfo1(log) - } - } return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err) } } diff --git a/tests/unit/scheduler-k3s-2.bats b/tests/unit/scheduler-k3s-2.bats index 4eec4820a..60734ad07 100644 --- a/tests/unit/scheduler-k3s-2.bats +++ b/tests/unit/scheduler-k3s-2.bats @@ -12,7 +12,6 @@ setup() { } teardown_() { - kubectl get pods -n cert-manager || true global_teardown dokku nginx:start uninstall_k3s || true From 78bde7f817b914cf780e8e7657074dc4980a40ea Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:15:08 -0400 Subject: [PATCH 20/28] feat: add more context for warning about not having an autoscaling config --- plugins/scheduler-k3s/functions.go | 2 +- tests/unit/scheduler-k3s-3.bats | 63 ++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 tests/unit/scheduler-k3s-3.bats diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 76f5991d7..10328d9d2 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -576,7 +576,7 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { } if !ok { - common.LogWarn(fmt.Sprintf("No autoscaling config found for %s", input.AppName)) + common.LogWarn(fmt.Sprintf("No autoscaling config found for %s:%s", input.AppName, input.ProcessType)) return ProcessAutoscaling{}, nil } diff --git a/tests/unit/scheduler-k3s-3.bats b/tests/unit/scheduler-k3s-3.bats new file mode 100644 index 000000000..a27ab8c17 --- /dev/null +++ b/tests/unit/scheduler-k3s-3.bats @@ -0,0 +1,63 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown_() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s) kustomize" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.replicas'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "3" +} + +inject_kustomization_yaml() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + mkdir -p "$APP_REPO_DIR/config/kustomize" + touch "$APP_REPO_DIR/config/kustomize/kustomization.yaml" + cat <"$APP_REPO_DIR/config/kustomize/kustomization.yaml" +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- patch: |- + - op: replace + path: /spec/replicas + value: 3 + target: + group: apps + version: v1 + kind: Deployment +EOF +} From 68f909b91d94cfcea75703511365e09c9acfe7d2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:16:49 -0400 Subject: [PATCH 21/28] fix: only process web-related templates when there is a web section embedded in the template --- plugins/scheduler-k3s/templates/chart/certificate.yaml | 4 ++-- plugins/scheduler-k3s/templates/chart/deployment.yaml | 10 +++++----- .../templates/chart/https-redirect-middleware.yaml | 4 ++-- .../scheduler-k3s/templates/chart/ingress-route.yaml | 4 ++-- plugins/scheduler-k3s/templates/chart/ingress.yaml | 6 +++--- .../templates/chart/keda-http-scaled-object.yaml | 6 +++--- .../chart/keda-interceptor-proxy-service.yaml | 6 +++--- .../templates/chart/keda-scaled-object.yaml | 4 ++-- plugins/scheduler-k3s/templates/chart/service.yaml | 4 ++-- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml index a16e03a8a..5065d77f8 100644 --- a/plugins/scheduler-k3s/templates/chart/certificate.yaml +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index 6d06ec008..4ffa5ba29 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -1,11 +1,11 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} {{- $mappings := dict }} -{{- if eq $processName "web" }} +{{- if hasKey $config "web" }} {{- range $pdx, $port_map := $config.web.port_maps }} {{- $mappings := set $mappings $port_map.name "true" }} {{- end }} @@ -61,7 +61,7 @@ spec: {{- range $config.args }} - {{ . }} {{- end }} - {{- if eq $processName "web" }} + {{- if hasKey $config "web" }} env: - name: PORT value: "{{ $.Values.global.network.primary_port }}" @@ -73,7 +73,7 @@ spec: image: {{ $.Values.global.image.name }} imagePullPolicy: Always name: {{ $.Values.global.app_name }}-{{ $processName }} - {{- if eq $processName "web" }} + {{- if hasKey $config "web" }} ports: {{- range $pdx, $port_map := $config.web.port_maps }} {{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }} diff --git a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml index d81397a53..b4b6bc810 100644 --- a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml +++ b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index 01bf6ff76..bdf492282 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index b416c3eab..2349d0292 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,11 +1,11 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} {{- $routeToKeda := false }} -{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} {{- if and ($.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }} {{- $routeToKeda = true }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml index be6890a96..2e1fea6db 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -1,11 +1,11 @@ {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} -{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} {{- if and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http") }} {{- $mappings := dict }} {{- range $pdx, $port_map := $config.web.port_maps }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml index 4c259c946..f7b76393b 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml @@ -1,11 +1,11 @@ {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} -{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} --- kind: Service apiVersion: v1 diff --git a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml index b6633556f..e57e6a799 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml @@ -1,7 +1,7 @@ {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml index 0b6a02417..f35967ab1 100644 --- a/plugins/scheduler-k3s/templates/chart/service.yaml +++ b/plugins/scheduler-k3s/templates/chart/service.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +# Skip processes that don't have a web section +{{- if not (hasKey $config "web") }} {{- continue }} {{- end }} From 2958f906e4b502d153decb76ef12e4263c3128db Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:28:40 -0400 Subject: [PATCH 22/28] chore: update and move comment in rendered template --- plugins/scheduler-k3s/templates/chart/certificate.yaml | 2 +- plugins/scheduler-k3s/templates/chart/deployment.yaml | 2 +- .../templates/chart/https-redirect-middleware.yaml | 2 +- plugins/scheduler-k3s/templates/chart/ingress-route.yaml | 2 +- plugins/scheduler-k3s/templates/chart/ingress.yaml | 2 +- .../scheduler-k3s/templates/chart/keda-http-scaled-object.yaml | 2 +- .../templates/chart/keda-interceptor-proxy-service.yaml | 2 +- plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml | 2 +- plugins/scheduler-k3s/templates/chart/service.yaml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml index 5065d77f8..077470ce8 100644 --- a/plugins/scheduler-k3s/templates/chart/certificate.yaml +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index 4ffa5ba29..a741804d1 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml index b4b6bc810..4a38d1291 100644 --- a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml +++ b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index bdf492282..fb137204e 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index 2349d0292..1b5cd1081 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml index 2e1fea6db..5907ead1e 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -1,7 +1,7 @@ {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml index f7b76393b..ade8549a2 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml @@ -1,7 +1,7 @@ {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml index e57e6a799..3f37603ac 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml @@ -1,7 +1,7 @@ {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml index f35967ab1..8980597da 100644 --- a/plugins/scheduler-k3s/templates/chart/service.yaml +++ b/plugins/scheduler-k3s/templates/chart/service.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip processes that don't have a web section {{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} From 553378085fae9b9688f9a495af65e183144f2efa Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:35:02 -0400 Subject: [PATCH 23/28] fix: ensure the rendered.yaml file is written into the secure filesystem --- plugins/scheduler-k3s/helm.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 98fedf630..c886a8020 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -495,8 +495,7 @@ func (p *KustomizeRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, return nil, fmt.Errorf("%s not found", konfig.DefaultKustomizationFileName()) } - renderedYamlPath := filepath.Join(p.KustomizeRootPath, "rendered.yaml") - if err := os.WriteFile(renderedYamlPath, renderedManifests.Bytes(), 0644); err != nil { + if err := fs.WriteFile(filepath.Join(p.KustomizeRootPath, "rendered.yaml"), renderedManifests.Bytes()); err != nil { return nil, fmt.Errorf("Error writing rendered.yaml: %w", err) } From 476cc6045f7f1bb5b01589089741344595d7e6ea Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:46:37 -0400 Subject: [PATCH 24/28] fix: add resources to injected template --- tests/unit/scheduler-k3s-3.bats | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/scheduler-k3s-3.bats b/tests/unit/scheduler-k3s-3.bats index a27ab8c17..d683f6cb0 100644 --- a/tests/unit/scheduler-k3s-3.bats +++ b/tests/unit/scheduler-k3s-3.bats @@ -50,6 +50,8 @@ inject_kustomization_yaml() { cat <"$APP_REPO_DIR/config/kustomize/kustomization.yaml" apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +resources: +- rendered.yaml patches: - patch: |- - op: replace From cd275138f33281c5822bfaeb86e20b4226ee7676 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:46:47 -0400 Subject: [PATCH 25/28] fix: actually use helper to inject template --- tests/unit/scheduler-k3s-3.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/scheduler-k3s-3.bats b/tests/unit/scheduler-k3s-3.bats index d683f6cb0..c6dba70c6 100644 --- a/tests/unit/scheduler-k3s-3.bats +++ b/tests/unit/scheduler-k3s-3.bats @@ -29,7 +29,7 @@ teardown_() { echo "status: $status" assert_success - run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP convert_to_dockerfile + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP inject_kustomization_yaml echo "output: $output" echo "status: $status" assert_success From 4aed599d8ccf1e047489c2a9fde432826823ee99 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:47:31 -0400 Subject: [PATCH 26/28] fix: do not run helm in dry-run mode when trace-mode is enabled --- plugins/scheduler-k3s/helm.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index c886a8020..e9dd9f97e 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -273,7 +273,6 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { client.CreateNamespace = true client.DryRun = false if os.Getenv("DOKKU_TRACE") == "1" { - client.DryRun = true client.PostRenderer = &DebugRenderer{ Renderer: &kustomizeRenderer, } @@ -397,7 +396,6 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { client.CleanupOnFail = true client.MaxHistory = 10 if os.Getenv("DOKKU_TRACE") == "1" { - client.DryRun = true client.PostRenderer = &DebugRenderer{ Renderer: &kustomizeRenderer, } From dd7589323e673d0bc36d5028756d60f52e332f43 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:47:46 -0400 Subject: [PATCH 27/28] feat: implement fetching the process-specific kustomize root path --- plugins/scheduler-k3s/functions.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 10328d9d2..16b9a1ab6 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1500,7 +1500,17 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { } func getProcessSpecificKustomizeRootPath(appName string) string { - return "" + if !hasKustomizeDirectory(appName) { + return "" + } + + directory := getKustomizeDirectory(appName) + processSpecificDirectory := fmt.Sprintf("%s.%s", directory, os.Getenv("DOKKU_PID")) + if common.DirectoryExists(processSpecificDirectory) { + return processSpecificDirectory + } + + return directory } func hasKustomizeDirectory(appName string) bool { From 245263d2ad10294febc97c7971cb3e6aa21b40a2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Jul 2025 20:58:18 -0400 Subject: [PATCH 28/28] docs: add documentation for Kustomize usage --- docs/deployment/schedulers/k3s.md | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 372ce4d70..5056deb2e 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -141,6 +141,7 @@ The k3s plugin provides a number of settings that can be used to managed deploym | Name | Description | Global Default | |-----------------------|---------------------------------------------------|--------------------| | `deploy-timeout` | Controls when app deploys will timeout in seconds | `300s` | +| `kustomize-root-path` | Controls the folder context from the deployed repository used for Kustomize | `config/kustomize` | | `image-pull-secrets` | Name of a kubernetes secret used to auth against a registry | Contents of `~/.docker/config.json` from Dokku server | | `namespace` | Controls the namespace used for resource creation | `default` | | `rollback-on-failure` | Whether to rollback failed deploys | `false` | @@ -365,7 +366,6 @@ dokku scheduler-k3s:ensure-charts Once the chart is configured, an `http` trigger can be specified like so: - ```json { "formation": { @@ -461,6 +461,30 @@ dokku scheduler-k3s:autoscaling-auth:report node-js-app --include-metadata Datadog datadogSite: us5.datadoghq.com ``` +### Integrating Kustomize + +Dokku supports integration with [Kustomize](https://kustomize.io/) to further customize the generated helm charts for app deployments. For example, a `config/kustomize/kustomization.yaml` file with the following contents will override the scale for each process deployed to `3`: + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- rendered.yaml +patches: +- patch: |- + - op: replace + path: /spec/replicas + value: 3 + target: + group: apps + version: v1 + kind: Deployment +``` + +When using Kustomize with Dokku, all Kustomize-files _must_ be placed in the `config/kustomize` folder, with a `kustomization.yaml` file being the entrypoint to Kustomize. Dokku will render the helm chart to a `rendered.yaml` file, and then execute Kustomize with the `config/kustomize` folder as the context. + +See the [Kustomize](https://kustomize.io/) website for more details on how to use Kustomize. + ### Using kubectl remotely > [!WARNING] @@ -515,7 +539,6 @@ dokku scheduler-k3s:set --global chart.cert-manager.version 1.13.3 > [!NOTE] > Properties follow dot-notation, and are expanded according to Helm's internal logic. See the [Helm documentation](https://helm.sh/docs/helm/helm_install/#helm-install) for `helm install` for further details. - To unset a chart property, omit the value from the `scheduler-k3s:set` call: ```shell