diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 372ce4d70..5056deb2e 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -141,6 +141,7 @@ The k3s plugin provides a number of settings that can be used to managed deploym | Name | Description | Global Default | |-----------------------|---------------------------------------------------|--------------------| | `deploy-timeout` | Controls when app deploys will timeout in seconds | `300s` | +| `kustomize-root-path` | Controls the folder context from the deployed repository used for Kustomize | `config/kustomize` | | `image-pull-secrets` | Name of a kubernetes secret used to auth against a registry | Contents of `~/.docker/config.json` from Dokku server | | `namespace` | Controls the namespace used for resource creation | `default` | | `rollback-on-failure` | Whether to rollback failed deploys | `false` | @@ -365,7 +366,6 @@ dokku scheduler-k3s:ensure-charts Once the chart is configured, an `http` trigger can be specified like so: - ```json { "formation": { @@ -461,6 +461,30 @@ dokku scheduler-k3s:autoscaling-auth:report node-js-app --include-metadata Datadog datadogSite: us5.datadoghq.com ``` +### Integrating Kustomize + +Dokku supports integration with [Kustomize](https://kustomize.io/) to further customize the generated helm charts for app deployments. For example, a `config/kustomize/kustomization.yaml` file with the following contents will override the scale for each process deployed to `3`: + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- rendered.yaml +patches: +- patch: |- + - op: replace + path: /spec/replicas + value: 3 + target: + group: apps + version: v1 + kind: Deployment +``` + +When using Kustomize with Dokku, all Kustomize-files _must_ be placed in the `config/kustomize` folder, with a `kustomization.yaml` file being the entrypoint to Kustomize. Dokku will render the helm chart to a `rendered.yaml` file, and then execute Kustomize with the `config/kustomize` folder as the context. + +See the [Kustomize](https://kustomize.io/) website for more details on how to use Kustomize. + ### Using kubectl remotely > [!WARNING] @@ -515,7 +539,6 @@ dokku scheduler-k3s:set --global chart.cert-manager.version 1.13.3 > [!NOTE] > Properties follow dot-notation, and are expanded according to Helm's internal logic. See the [Helm documentation](https://helm.sh/docs/helm/helm_install/#helm-install) for `helm install` for further details. - To unset a chart property, omit the value from the `scheduler-k3s:set` call: ```shell diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index 449103c2c..870e3ba05 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall -TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop +TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 942d5c4f2..16b9a1ab6 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -576,7 +576,7 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { } if !ok { - common.LogWarn(fmt.Sprintf("No autoscaling config found for %s", input.AppName)) + common.LogWarn(fmt.Sprintf("No autoscaling config found for %s:%s", input.AppName, input.ProcessType)) return ProcessAutoscaling{}, nil } @@ -1190,6 +1190,28 @@ func getGlobalLetsencryptEmailStag() string { return common.PropertyGetDefault("scheduler-k3s", "--global", "letsencrypt-email-stag", "") } +func getKustomizeDirectory(appName string) string { + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName) + return filepath.Join(directory, "kustomization") +} + +func getKustomizeRootPath(appName string) string { + return common.PropertyGetDefault("scheduler-k3s", appName, "kustomize-root-path", "") +} + +func getGlobalKustomizeRootPath() string { + return common.PropertyGetDefault("scheduler-k3s", "--global", "kustomize-root-path", "config/kustomize") +} + +func getComputedKustomizeRootPath(appName string) string { + kustomizeRootPath := getKustomizeRootPath(appName) + if kustomizeRootPath == "" { + kustomizeRootPath = getGlobalKustomizeRootPath() + } + + return kustomizeRootPath +} + func getNamespace(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "") } @@ -1477,6 +1499,34 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { }, nil } +func getProcessSpecificKustomizeRootPath(appName string) string { + if !hasKustomizeDirectory(appName) { + return "" + } + + directory := getKustomizeDirectory(appName) + processSpecificDirectory := fmt.Sprintf("%s.%s", directory, os.Getenv("DOKKU_PID")) + if common.DirectoryExists(processSpecificDirectory) { + return processSpecificDirectory + } + + return directory +} + +func hasKustomizeDirectory(appName string) bool { + directory := getKustomizeDirectory(appName) + + if common.DirectoryExists(fmt.Sprintf("%s.%s.missing", directory, os.Getenv("DOKKU_PID"))) { + return false + } + + if common.DirectoryExists(fmt.Sprintf("%s.%s", directory, os.Getenv("DOKKU_PID"))) { + return true + } + + return common.DirectoryExists(directory) +} + func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldInstall func(HelmChart) bool) error { for _, repo := range HelmRepositories { helmAgent, err := NewHelmAgent("default", DeployLogPrinter) diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index 2e7f2df80..bf48e9f6f 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -11,6 +11,7 @@ require ( github.com/dokku/dokku/plugins/logs v0.0.0-20250618161309-8d0c35f1333c github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-20250618161309-8d0c35f1333c github.com/fatih/color v1.18.0 + github.com/fluxcd/pkg/kustomize v1.18.0 github.com/go-openapi/jsonpointer v0.21.1 github.com/go-resty/resty/v2 v2.16.5 github.com/gofrs/flock v0.12.1 @@ -30,6 +31,7 @@ require ( k8s.io/kubernetes v1.33.2 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 mvdan.cc/sh/v3 v3.12.0 + sigs.k8s.io/kustomize/api v0.20.0 ) require ( @@ -168,7 +170,6 @@ require ( sigs.k8s.io/controller-runtime v0.21.0 // indirect sigs.k8s.io/gateway-api v1.3.0 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect - sigs.k8s.io/kustomize/api v0.20.0 // indirect sigs.k8s.io/kustomize/kyaml v0.20.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum index 46f0a044d..08511dbb7 100644 --- a/plugins/scheduler-k3s/go.sum +++ b/plugins/scheduler-k3s/go.sum @@ -89,6 +89,8 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fluxcd/pkg/kustomize v1.18.0 h1:wWK+qYwmBmba3N3VAqZ9ijnfVGGaIjcaHWo033URZTw= +github.com/fluxcd/pkg/kustomize v1.18.0/go.mod h1:Ij9722MdWIE6B1EPg2ZJUf6npycgfRfN4Lohi7D/Kic= github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 8f007cdf1..e9dd9f97e 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -14,6 +14,7 @@ import ( "unicode" "github.com/dokku/dokku/plugins/common" + "github.com/fluxcd/pkg/kustomize/filesys" "github.com/gofrs/flock" "gopkg.in/yaml.v3" "helm.sh/helm/v3/pkg/action" @@ -21,9 +22,13 @@ import ( "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/kube" + "helm.sh/helm/v3/pkg/postrender" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/storage/driver" + "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/api/types" ) var DevNullPrinter = func(format string, v ...interface{}) {} @@ -46,16 +51,37 @@ var DeployLogPrinter = func(format string, v ...interface{}) { } } +// ChartInput is the input for the InstallOrUpgradeChart function type ChartInput struct { - ChartPath string - Namespace string - ReleaseName string - RepoURL string + // ChartPath is the path to the chart to install or upgrade + ChartPath string + + // KustomizeRootPath is the path to the kustomize root path to use + KustomizeRootPath string + + // Namespace is the namespace to install or upgrade the chart in + Namespace string + + // ReleaseName is the name of the release to install or upgrade + ReleaseName string + + // RepoURL is the URL of the chart repository to install or upgrade the chart from + RepoURL string + + // RollbackOnFailure is whether to rollback on failure RollbackOnFailure bool - Timeout time.Duration - Wait bool - Version string - Values map[string]interface{} + + // Timeout is the timeout for the install or upgrade + Timeout time.Duration + + // Wait is whether to wait for the install or upgrade to complete + Wait bool + + // Version is the version of the chart to install or upgrade + Version string + + // Values is the values to pass to the chart + Values map[string]interface{} } type Release struct { @@ -237,11 +263,22 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { input.Values = map[string]interface{}{} } + kustomizeRenderer := KustomizeRenderer{ + KustomizeRootPath: input.KustomizeRootPath, + } + client := action.NewInstall(h.Configuration) client.Atomic = false client.ChartPathOptions = action.ChartPathOptions{} client.CreateNamespace = true client.DryRun = false + if os.Getenv("DOKKU_TRACE") == "1" { + client.PostRenderer = &DebugRenderer{ + Renderer: &kustomizeRenderer, + } + } else { + client.PostRenderer = &kustomizeRenderer + } client.Namespace = namespace client.ReleaseName = input.ReleaseName client.Timeout = input.Timeout @@ -333,16 +370,6 @@ func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) { return releases, nil } -type DebugRenderer struct { -} - -func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { - for _, line := range strings.Split(renderedManifests.String(), "\n") { - common.LogWarn(line) - } - return renderedManifests, nil -} - func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { namespace := input.Namespace if namespace == "" { @@ -359,14 +386,21 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { input.Values = map[string]interface{}{} } + kustomizeRenderer := KustomizeRenderer{ + KustomizeRootPath: input.KustomizeRootPath, + } + client := action.NewUpgrade(h.Configuration) client.Atomic = input.RollbackOnFailure client.ChartPathOptions = action.ChartPathOptions{} client.CleanupOnFail = true client.MaxHistory = 10 if os.Getenv("DOKKU_TRACE") == "1" { - client.DryRun = true - client.PostRenderer = &DebugRenderer{} + client.PostRenderer = &DebugRenderer{ + Renderer: &kustomizeRenderer, + } + } else { + client.PostRenderer = &kustomizeRenderer } client.Namespace = namespace client.Timeout = input.Timeout @@ -413,3 +447,71 @@ func (h *HelmAgent) UninstallChart(releaseName string) error { return nil } + +type DebugRenderer struct { + Renderer postrender.PostRenderer +} + +func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { + renderedManifests, err := p.Renderer.Run(renderedManifests) + if err != nil { + return nil, err + } + + for _, line := range strings.Split(renderedManifests.String(), "\n") { + common.LogWarn(line) + } + return renderedManifests, nil +} + +type KustomizeRenderer struct { + KustomizeRootPath string +} + +func (p *KustomizeRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { + if p.KustomizeRootPath == "" { + return renderedManifests, nil + } + + if !common.DirectoryExists(p.KustomizeRootPath) { + return renderedManifests, nil + } + + fs, err := filesys.MakeFsOnDiskSecureBuild(p.KustomizeRootPath) + if err != nil { + return nil, fmt.Errorf("Error creating filesystem: %w", err) + } + + var kfile string + for _, f := range konfig.RecognizedKustomizationFileNames() { + if kf := filepath.Join(p.KustomizeRootPath, f); fs.Exists(kf) { + kfile = kf + break + } + } + if kfile == "" { + return nil, fmt.Errorf("%s not found", konfig.DefaultKustomizationFileName()) + } + + if err := fs.WriteFile(filepath.Join(p.KustomizeRootPath, "rendered.yaml"), renderedManifests.Bytes()); err != nil { + return nil, fmt.Errorf("Error writing rendered.yaml: %w", err) + } + + buildOptions := &krusty.Options{ + LoadRestrictions: types.LoadRestrictionsNone, + PluginConfig: types.DisabledPluginConfig(), + } + + k := krusty.MakeKustomizer(buildOptions) + m, err := k.Run(fs, p.KustomizeRootPath) + if err != nil { + return nil, err + } + + resources, err := m.AsYaml() + if err != nil { + return nil, err + } + + return bytes.NewBuffer(resources), nil +} diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index 494d5ab9a..491e9174a 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -416,6 +416,35 @@ func (k KubernetesClient) ExecCommand(ctx context.Context, input ExecCommandInpu }) } +// GetPodLogsInput contains all the information needed to get the logs for a Kubernetes pod +type GetLogsInput struct { + // Name is the Kubernetes pod name + Name string + + // Namespace is the Kubernetes namespace + Namespace string +} + +// GetLogs gets the logs for a Kubernetes pod +func (k KubernetesClient) GetLogs(ctx context.Context, input GetLogsInput) ([]byte, error) { + logOptions := corev1.PodLogOptions{} + + request := k.Client.CoreV1().Pods(input.Namespace).GetLogs(input.Name, &logOptions) + + readCloser, err := request.Stream(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get logs: %w", err) + } + defer readCloser.Close() + + bytes, err := io.ReadAll(readCloser) + if err != nil { + return nil, fmt.Errorf("failed to read logs: %w", err) + } + + return bytes, nil +} + // GetNodeInput contains all the information needed to get a Kubernetes node type GetNodeInput struct { // Name is the Kubernetes node name diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index d480969b6..d1bb6d2de 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -28,6 +28,9 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-global-ingress-class": reportGlobalIngressClass, "--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd, "--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag, + "--scheduler-k3s-computed-kustomize-root-path": reportComputedKustomizeRootPath, + "--scheduler-k3s-kustomize-root-path": reportKustomizeRootPath, + "--scheduler-k3s-global-kustomize-root-path": reportGlobalKustomizeRootPath, "--scheduler-k3s-computed-namespace": reportComputedNamespace, "--scheduler-k3s-namespace": reportNamespace, "--scheduler-k3s-global-namespace": reportGlobalNamespace, @@ -148,6 +151,18 @@ func reportGlobalLetsencryptEmailStag(appName string) string { return getGlobalLetsencryptEmailStag() } +func reportComputedKustomizeRootPath(appName string) string { + return getComputedKustomizeRootPath(appName) +} + +func reportKustomizeRootPath(appName string) string { + return getKustomizeRootPath(appName) +} + +func reportGlobalKustomizeRootPath(appName string) string { + return getGlobalKustomizeRootPath() +} + func reportComputedNamespace(appName string) string { return getComputedNamespace(appName) } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 9a37a78bd..fbf29a7bc 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -20,6 +20,7 @@ var ( DefaultProperties = map[string]string{ "deploy-timeout": "", "letsencrypt-server": "", + "kustomize-root-path": "", "image-pull-secrets": "", "namespace": "", "rollback-on-failure": "", @@ -33,6 +34,7 @@ var ( "ingress-class": true, "kube-context": true, "kubeconfig-path": true, + "kustomize-root-path": true, "letsencrypt-server": true, "letsencrypt-email-prod": true, "letsencrypt-email-stag": true, diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go index 4092da096..692c62e1f 100644 --- a/plugins/scheduler-k3s/src/triggers/triggers.go +++ b/plugins/scheduler-k3s/src/triggers/triggers.go @@ -22,6 +22,13 @@ func main() { var err error switch trigger { + case "core-post-deploy": + appName := flag.Arg(0) + err = scheduler_k3s.TriggerCorePostDeploy(appName) + case "core-post-extract": + appName := flag.Arg(0) + sourceWorkDir := flag.Arg(1) + err = scheduler_k3s.TriggerCorePostExtract(appName, sourceWorkDir) case "install": err = scheduler_k3s.TriggerInstall() case "post-app-clone-setup": @@ -32,6 +39,9 @@ func main() { oldAppName := flag.Arg(0) newAppName := flag.Arg(1) err = scheduler_k3s.TriggerPostAppRenameSetup(oldAppName, newAppName) + case "post-create": + appName := flag.Arg(0) + err = scheduler_k3s.TriggerPostCreate(appName) case "post-delete": appName := flag.Arg(0) err = scheduler_k3s.TriggerPostDelete(appName) diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml index a16e03a8a..077470ce8 100644 --- a/plugins/scheduler-k3s/templates/chart/certificate.yaml +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index 6d06ec008..a741804d1 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -1,11 +1,11 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} {{- $mappings := dict }} -{{- if eq $processName "web" }} +{{- if hasKey $config "web" }} {{- range $pdx, $port_map := $config.web.port_maps }} {{- $mappings := set $mappings $port_map.name "true" }} {{- end }} @@ -61,7 +61,7 @@ spec: {{- range $config.args }} - {{ . }} {{- end }} - {{- if eq $processName "web" }} + {{- if hasKey $config "web" }} env: - name: PORT value: "{{ $.Values.global.network.primary_port }}" @@ -73,7 +73,7 @@ spec: image: {{ $.Values.global.image.name }} imagePullPolicy: Always name: {{ $.Values.global.app_name }}-{{ $processName }} - {{- if eq $processName "web" }} + {{- if hasKey $config "web" }} ports: {{- range $pdx, $port_map := $config.web.port_maps }} {{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }} diff --git a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml index d81397a53..4a38d1291 100644 --- a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml +++ b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index 01bf6ff76..fb137204e 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index b416c3eab..1b5cd1081 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,11 +1,11 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} {{- $routeToKeda := false }} -{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} {{- if and ($.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }} {{- $routeToKeda = true }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml index be6890a96..5907ead1e 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -1,11 +1,11 @@ {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} -{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} {{- if and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http") }} {{- $mappings := dict }} {{- range $pdx, $port_map := $config.web.port_maps }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml index 4c259c946..ade8549a2 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml @@ -1,11 +1,11 @@ {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} -{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} --- kind: Service apiVersion: v1 diff --git a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml index b6633556f..3f37603ac 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml @@ -1,7 +1,7 @@ {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}} {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml index 0b6a02417..8980597da 100644 --- a/plugins/scheduler-k3s/templates/chart/service.yaml +++ b/plugins/scheduler-k3s/templates/chart/service.yaml @@ -1,6 +1,6 @@ {{- range $processName, $config := .Values.processes }} -# Skip cron jobs -{{- if and (hasKey $config "cron") (hasKey $config.cron "id") }} +{{- if not (hasKey $config "web") }} +# Skip {{ $processName }} as it doesn't have a web section {{- continue }} {{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index a228ac048..7c0a9f8c4 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -32,11 +32,50 @@ import ( "k8s.io/kubernetes/pkg/client/conditions" ) +// TriggerCorePostDeploy moves a configured kustomize root path to be in the app root dir +func TriggerCorePostDeploy(appName string) error { + return common.CorePostDeploy(common.CorePostDeployInput{ + AppName: appName, + Destination: common.GetAppDataDirectory("scheduler-k3s", appName), + PluginName: "scheduler-k3s", + ExtractedPaths: []common.CorePostDeployPath{ + {Path: "kustomization", IsDirectory: true}, + }, + }) + + return nil +} + +// TriggerCorePostExtract moves a configured kustomize root path to be in the app root dir +func TriggerCorePostExtract(appName string, sourceWorkDir string) error { + destination := common.GetAppDataDirectory("scheduler-k3s", appName) + kustomizeRootPath := getComputedKustomizeRootPath(appName) + return common.CorePostExtract(common.CorePostExtractInput{ + AppName: appName, + Destination: destination, + PluginName: "scheduler-k3s", + SourceWorkDir: sourceWorkDir, + ToExtract: []common.CorePostExtractToExtract{ + { + Path: kustomizeRootPath, + IsDirectory: true, + Name: "config/kustomize", + Destination: "kustomization", + }, + }, + }) +} + // TriggerInstall runs the install step for the scheduler-k3s plugin func TriggerInstall() error { if err := common.PropertySetup("scheduler-k3s"); err != nil { return fmt.Errorf("Unable to install the scheduler-k3s plugin: %s", err.Error()) } + + if err := common.SetupAppData("scheduler-k3s"); err != nil { + return err + } + return nil } @@ -63,6 +102,11 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error { return nil } +// TriggerPostCreate creates the scheduler-k3s data directory +func TriggerPostCreate(appName string) error { + return common.CreateAppDataDirectory("scheduler-k3s", appName) +} + // TriggerPostDelete destroys the scheduler-k3s data for a given app container func TriggerPostDelete(appName string) error { dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName) @@ -704,9 +748,15 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } } + kustomizeRootPath := "" + if hasKustomizeDirectory(appName) { + kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName) + } + common.LogInfo2(fmt.Sprintf("Installing %s", appName)) err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{ ChartPath: chartPath, + KustomizeRootPath: kustomizeRootPath, Namespace: namespace, ReleaseName: appName, RollbackOnFailure: allowRollbacks, diff --git a/tests/unit/scheduler-k3s-3.bats b/tests/unit/scheduler-k3s-3.bats new file mode 100644 index 000000000..c6dba70c6 --- /dev/null +++ b/tests/unit/scheduler-k3s-3.bats @@ -0,0 +1,65 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown_() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s) kustomize" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP inject_kustomization_yaml + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.replicas'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "3" +} + +inject_kustomization_yaml() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + mkdir -p "$APP_REPO_DIR/config/kustomize" + touch "$APP_REPO_DIR/config/kustomize/kustomization.yaml" + cat <"$APP_REPO_DIR/config/kustomize/kustomization.yaml" +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- rendered.yaml +patches: +- patch: |- + - op: replace + path: /spec/replicas + value: 3 + target: + group: apps + version: v1 + kind: Deployment +EOF +}