Merge pull request #8729 from dokku/scheduler-k3s-diff-subcommand

Add scheduler-k3s:preview subcommand

docs/deployment/schedulers/k3s.md

@@ -17,6 +17,7 @@ scheduler-k3s:ensure-charts                         # Ensures the k3s charts are
 scheduler-k3s:initialize                            # Initializes a cluster
 scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Set or clear a label for a given app/process-type/resource-type combination
 scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Displays a scheduler-k3s labels report for one or more apps
+scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded] # Displays a diff between the current and next deployment for an app
 scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] Adds a node profile to the k3s cluster
 scheduler-k3s:profiles:list [--format json|stdout]  # Lists all node profiles in the k3s cluster
 scheduler-k3s:profiles:remove <profile>             # Removes a node profile from the k3s cluster
@@ -236,6 +237,38 @@ The global default value may be set by passing an empty value for the option.
 dokku scheduler-k3s:set --global deploy-timeout
 ```
 
+### Previewing deployment changes
+
+Before triggering a deploy, the `scheduler-k3s:preview` command can be used to display a unified diff between the manifests currently stored in the live Helm release for an app and the manifests that the next deploy would roll out:
+
+```shell
+dokku scheduler-k3s:preview node-js-app
+```
+
+A clean redeploy with no property changes produces no output. After modifying a property that affects the main app chart (for example `dokku resource:limit --memory 256m node-js-app`), the command shows the unified diff for each changed resource. For an app that has never been deployed, the entire proposed manifest is rendered as added lines.
+
+By default each change is shown with 3 lines of surrounding context, git-diff style. The `--context` flag overrides this; pass `-1` to render the full resource around every change:
+
+```shell
+dokku scheduler-k3s:preview node-js-app --context 0
+dokku scheduler-k3s:preview node-js-app --context -1
+```
+
+By default any `kind: Secret` resources have their `data` values redacted so secret bytes never appear in terminal scrollback or CI logs. Two flags adjust this behavior:
+
+```shell
+dokku scheduler-k3s:preview node-js-app --show-secrets
+dokku scheduler-k3s:preview node-js-app --show-secrets-decoded
+```
+
+`--show-secrets` keeps the raw base64-encoded values, and `--show-secrets-decoded` renders them base64-decoded. In current dokku, the only Secret objects in the main app chart come from KEDA autoscaling auth resources configured via [`scheduler-k3s:autoscaling-auth:set`](#workload-autoscaling-authentication); for apps without autoscaling auth, these flags are no-ops.
+
+> [!IMPORTANT]
+> The preview only covers the main app helm release (the release named after the app). Dokku installs three auxiliary helm releases per app for config-var storage, image-pull credentials, and TLS certificates, and changes to those releases do **not** appear in this preview. In particular, a `dokku config:set` change followed by `scheduler-k3s:preview` will show no diff because the new config-var value lives in the separate `config-<app>` helm release.
+
+> [!NOTE]
+> Helm renders the proposed manifest using a client-side dry-run. The `lookup` template function returns empty values in this mode, so charts overridden via `scheduler-k3s:charts:set` that depend on `lookup` may render differently here than at actual deploy time. Dokku's bundled chart templates do not use `lookup`.
+
 ### Exposing services on the network
 
 Dokku will automatically expose the `web` process as a Kubernetes Service, with all others being treated as background processes. In some cases, it may be useful to have other processes exposed as Kubernetes Service objects so as to segregate internal http endpoints from public http endpoints. This can be done by modifying the `app.json` Formation entry for your process type.

plugins/scheduler-k3s/Makefile

@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
+SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
 TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s

plugins/scheduler-k3s/chart.go

@@ -0,0 +1,644 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	appjson "github.com/dokku/dokku/plugins/app-json"
+	"github.com/dokku/dokku/plugins/common"
+	"github.com/dokku/dokku/plugins/config"
+	"github.com/dokku/dokku/plugins/cron"
+	"github.com/dokku/dokku/plugins/registry"
+	"github.com/gosimple/slug"
+	"github.com/kballard/go-shellquote"
+)
+
+// BuildResult is the materialized output of BuildAppChart. It points at a
+// temporary chart directory on disk; the caller is responsible for removing
+// it once Helm is done with it.
+type BuildResult struct {
+	ChartDir          string
+	ChartPath         string
+	ReleaseName       string
+	Namespace         string
+	Timeout           time.Duration
+	RollbackOnFailure bool
+	KustomizeRootPath string
+}
+
+// BuildOptions configures non-default behavior for BuildAppChart.
+type BuildOptions struct {
+	// OverrideDeploymentID, when non-zero, replaces the timestamp-derived
+	// deployment id. CommandPreview uses this to reuse the currently-deployed
+	// release's id so app.kubernetes.io/version annotation churn does not
+	// appear in the diff.
+	OverrideDeploymentID int64
+
+	// AllowMissingImage, when true, lets BuildAppChart substitute a
+	// placeholder image string when no real image is available for the app
+	// yet (e.g. a never-deployed app being previewed). The deploy path
+	// leaves this false so a missing image still errors.
+	AllowMissingImage bool
+}
+
+// BuildAppChart constructs a Helm chart on disk that reflects dokku's configured
+// state for the given app and image tag. It reads from the Kubernetes cluster
+// where required (for KEDA values and existing cron-job suffixes) but performs
+// no writes to the cluster - no namespace, secrets, ingresses, or helm releases
+// are created or modified.
+//
+// The returned BuildResult holds the absolute chart path and the parameters a
+// subsequent Helm Install/Upgrade (dry-run or real) needs. The caller must
+// os.RemoveAll(result.ChartDir) when finished.
+func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOptions) (BuildResult, error) {
+	var result BuildResult
+
+	processesRes, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Trigger: "ps-current-scale",
+		Args:    []string{appName},
+	})
+	if err != nil {
+		return result, err
+	}
+
+	processes, err := common.ParseScaleOutput(processesRes.StdoutBytes())
+	if err != nil {
+		return result, err
+	}
+
+	namespace := getComputedNamespace(appName)
+
+	image, err := common.GetDeployingAppImageName(appName, imageTag, "")
+	if err != nil {
+		if !opts.AllowMissingImage {
+			return result, fmt.Errorf("Error getting deploying app image name: %w", err)
+		}
+		image = fmt.Sprintf("dokku/%s:not-yet-deployed", appName)
+	}
+
+	deployTimeout := getComputedDeployTimeout(appName)
+	if _, err := strconv.Atoi(deployTimeout); err == nil {
+		deployTimeout = fmt.Sprintf("%ss", deployTimeout)
+	}
+	timeoutDuration, err := time.ParseDuration(deployTimeout)
+	if err != nil {
+		return result, fmt.Errorf("Error parsing deploy timeout duration: %w", err)
+	}
+
+	deployRollback := getComputedRollbackOnFailure(appName)
+	allowRollbacks, err := strconv.ParseBool(deployRollback)
+	if err != nil {
+		return result, fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err)
+	}
+
+	imageSourceType := "dockerfile"
+	if common.IsImageCnbBased(image) {
+		imageSourceType = "pack"
+	} else if common.IsImageHerokuishBased(image, appName) {
+		imageSourceType = "herokuish"
+	}
+
+	env, err := config.LoadMergedAppEnv(appName)
+	if err != nil {
+		return result, fmt.Errorf("Error loading environment for deployment: %w", err)
+	}
+
+	importedCertExists := false
+	if HasImportedTLSCert(appName) {
+		exists, err := TLSSecretExists(ctx, appName)
+		if err == nil && exists {
+			importedCertExists = true
+		}
+	}
+
+	tlsEnabled := false
+	issuerName := ""
+	useImportedCert := false
+
+	if importedCertExists {
+		tlsEnabled = true
+		useImportedCert = true
+	} else {
+		server := getComputedLetsencryptServer(appName)
+		letsencryptEmailStag := getGlobalLetsencryptEmailStag()
+		letsencryptEmailProd := getGlobalLetsencryptEmailProd()
+
+		switch server {
+		case "prod", "production":
+			issuerName = "letsencrypt-prod"
+			tlsEnabled = letsencryptEmailProd != ""
+		case "stag", "staging":
+			issuerName = "letsencrypt-stag"
+			tlsEnabled = letsencryptEmailStag != ""
+		case "false":
+			issuerName = ""
+			tlsEnabled = false
+		default:
+			return result, fmt.Errorf("Invalid letsencrypt server config: %s", server)
+		}
+	}
+
+	chartDir, err := os.MkdirTemp("", "dokku-chart-")
+	if err != nil {
+		return result, fmt.Errorf("Error creating chart directory: %w", err)
+	}
+
+	// From this point on, any error returns require os.RemoveAll(chartDir).
+	cleanup := func(retErr error) (BuildResult, error) {
+		os.RemoveAll(chartDir)
+		return result, retErr
+	}
+
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
+		return cleanup(fmt.Errorf("Error creating chart templates directory: %w", err))
+	}
+
+	deploymentId := opts.OverrideDeploymentID
+	if deploymentId == 0 {
+		deploymentId = time.Now().Unix()
+	}
+	imagePullSecrets := getComputedImagePullSecrets(appName)
+	if imagePullSecrets == "" {
+		dockerConfigPath := filepath.Join(registry.GetComputedAppRegistryConfigDir(appName), "config.json")
+		if fi, err := os.Stat(dockerConfigPath); err == nil && !fi.IsDir() {
+			imagePullSecrets = GetImagePullSecretName(appName)
+		}
+	}
+
+	globalTemplateFiles := []string{"service-account"}
+	for _, templateName := range globalTemplateFiles {
+		b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
+		if err != nil {
+			return cleanup(fmt.Errorf("Error reading %s template: %w", templateName, err))
+		}
+
+		filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
+		if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
+			return cleanup(fmt.Errorf("Error writing %s template: %w", templateName, err))
+		}
+
+		if os.Getenv("DOKKU_TRACE") == "1" {
+			common.CatFile(filename)
+		}
+	}
+
+	portMaps, err := getPortMaps(appName)
+	if err != nil {
+		return cleanup(fmt.Errorf("Error getting port mappings for deployment: %w", err))
+	}
+
+	primaryPort := int32(5000)
+	primaryServicePort := int32(80)
+	for _, portMap := range portMaps {
+		primaryPort = portMap.ContainerPort
+		primaryServicePort = portMap.HostPort
+		if primaryPort != 0 {
+			break
+		}
+	}
+
+	appJSON, err := appjson.GetAppJSON(appName)
+	if err != nil {
+		return cleanup(fmt.Errorf("Error getting app.json for deployment: %w", err))
+	}
+
+	workingDir := common.GetWorkingDir(appName, image)
+
+	cronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
+	if err != nil {
+		return cleanup(fmt.Errorf("Error fetching cron tasks: %w", err))
+	}
+
+	domains := []string{}
+	if _, ok := processes["web"]; ok {
+		_, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+			Trigger:     "domains-vhost-enabled",
+			Args:        []string{appName},
+			StreamStdio: true,
+		})
+		if err == nil {
+			results, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+				Trigger: "domains-list",
+				Args:    []string{appName},
+			})
+			if err != nil {
+				return cleanup(fmt.Errorf("Error getting domains for deployment: %w", err))
+			}
+
+			for _, domain := range strings.Split(results.StdoutContents(), "\n") {
+				domain = strings.TrimSpace(domain)
+				if domain != "" {
+					domains = append(domains, domain)
+				}
+			}
+		}
+	}
+
+	chart := &Chart{
+		ApiVersion: "v2",
+		AppVersion: "1.0.0",
+		Name:       appName,
+		Icon:       "https://dokku.com/assets/dokku-logo.svg",
+		Version:    fmt.Sprintf("0.0.%d", deploymentId),
+	}
+
+	if err := writeYaml(WriteYamlInput{
+		Object: chart,
+		Path:   filepath.Join(chartDir, "Chart.yaml"),
+	}); err != nil {
+		return cleanup(fmt.Errorf("Error writing chart: %w", err))
+	}
+
+	globalAnnotations, err := getGlobalAnnotations(appName)
+	if err != nil {
+		return cleanup(fmt.Errorf("Error getting global annotations: %w", err))
+	}
+
+	globalLabels, err := getGlobalLabel(appName)
+	if err != nil {
+		return cleanup(fmt.Errorf("Error getting global labels: %w", err))
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return cleanup(fmt.Errorf("Error creating kubernetes client: %w", err))
+	}
+
+	if err := clientset.Ping(); err != nil {
+		return cleanup(fmt.Errorf("kubernetes api not available: %w", err))
+	}
+
+	kedaValues, err := getKedaValues(ctx, clientset, appName)
+	if err != nil {
+		return cleanup(fmt.Errorf("Error getting keda values: %w", err))
+	}
+
+	securityContext, err := getSecurityContext(appName, "deploy")
+	if err != nil {
+		return cleanup(fmt.Errorf("Error getting security context: %w", err))
+	}
+
+	values := &AppValues{
+		Global: GlobalValues{
+			Annotations:  globalAnnotations,
+			AppName:      appName,
+			DeploymentID: fmt.Sprint(deploymentId),
+			Keda:         kedaValues,
+			Image: GlobalImage{
+				ImagePullSecrets: imagePullSecrets,
+				Name:             image,
+				Type:             imageSourceType,
+				WorkingDir:       workingDir,
+			},
+			Labels:    globalLabels,
+			Namespace: namespace,
+			Network: GlobalNetwork{
+				IngressClass:       getComputedIngressClass(),
+				PrimaryPort:        primaryPort,
+				PrimaryServicePort: primaryServicePort,
+			},
+			SecurityContext: securityContext,
+		},
+		Processes: map[string]ProcessValues{},
+	}
+
+	if len(kedaValues.Authentications) > 0 {
+		templateFiles := []string{"keda-secret", "keda-trigger-authentication"}
+		for _, templateName := range templateFiles {
+			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
+			if err != nil {
+				return cleanup(fmt.Errorf("Error reading %s template: %w", templateName, err))
+			}
+
+			filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
+			if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
+				return cleanup(fmt.Errorf("Error writing %s template: %w", templateName, err))
+			}
+
+			if os.Getenv("DOKKU_TRACE") == "1" {
+				common.CatFile(filename)
+			}
+		}
+	}
+
+	processVolumes := []ProcessVolume{}
+	if shmSize := getComputedShmSize(appName); shmSize != "" {
+		processVolumes = append(processVolumes, ProcessVolume{
+			Name:      "shmem",
+			MountPath: "/dev/shm",
+			EmptyDir: &ProcessVolumeEmptyDir{
+				Medium:    "Memory",
+				SizeLimit: shmSize,
+			},
+		})
+	}
+
+	deployMountPairs, err := LoadAppMounts(appName, "deploy")
+	if err != nil {
+		return cleanup(fmt.Errorf("error loading storage app mounts: %w", err))
+	}
+	deployVolumes, err := ToProcessVolumes(deployMountPairs)
+	if err != nil {
+		return cleanup(err)
+	}
+	processVolumes = append(processVolumes, deployVolumes...)
+
+	for processType, processCount := range processes {
+		healthchecks, ok := appJSON.Healthchecks[processType]
+		if !ok {
+			healthchecks = []appjson.Healthcheck{}
+		}
+		processHealthchecks := getProcessHealtchecks(healthchecks, primaryPort)
+
+		startCommand, err := getStartCommand(StartCommandInput{
+			AppName:         appName,
+			ProcessType:     processType,
+			ImageSourceType: imageSourceType,
+			Port:            primaryPort,
+			Env:             env.Map(),
+		})
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting start command for deployment: %w", err))
+		}
+		args := startCommand.Command
+
+		processResources, err := getProcessResources(appName, processType)
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting process resources: %w", err))
+		}
+
+		annotations, err := getAnnotations(appName, processType)
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting process annotations: %w", err))
+		}
+
+		labels, err := getLabels(appName, processType)
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting process labels: %w", err))
+		}
+
+		autoscaling, err := getAutoscaling(GetAutoscalingInput{
+			AppName:     appName,
+			ProcessType: processType,
+			Replicas:    int(processCount),
+			KedaValues:  kedaValues,
+		})
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting autoscaling: %w", err))
+		}
+
+		processValues := ProcessValues{
+			Annotations:  annotations,
+			Autoscaling:  autoscaling,
+			Args:         args,
+			Healthchecks: processHealthchecks,
+			Labels:       labels,
+			ProcessType:  ProcessType_Worker,
+			Replicas:     int32(processCount),
+			Resources:    processResources,
+			Volumes:      processVolumes,
+		}
+
+		if processType == "web" {
+			sort.Strings(domains)
+			domainValues := []ProcessDomains{}
+			for _, domain := range domains {
+				domainValues = append(domainValues, ProcessDomains{
+					Name: domain,
+					Slug: slug.Make(domain),
+				})
+			}
+
+			processValues.Web = ProcessWeb{
+				Domains:  domainValues,
+				PortMaps: []ProcessPortMap{},
+				TLS: ProcessTls{
+					Enabled:         tlsEnabled,
+					IssuerName:      issuerName,
+					UseImportedCert: useImportedCert,
+				},
+			}
+
+			processValues.ProcessType = ProcessType_Web
+			for _, portMap := range portMaps {
+				protocol := PortmapProtocol_TCP
+				if portMap.Scheme == "udp" {
+					protocol = PortmapProtocol_UDP
+				}
+
+				processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
+					ContainerPort: portMap.ContainerPort,
+					HostPort:      portMap.HostPort,
+					Name:          portMap.String(),
+					Protocol:      protocol,
+					Scheme:        portMap.Scheme,
+				})
+			}
+
+			for _, portMap := range processValues.Web.PortMaps {
+				_, httpOk := portMaps[fmt.Sprintf("http-80-%d", portMap.ContainerPort)]
+				_, httpsOk := portMaps[fmt.Sprintf("https-443-%d", portMap.ContainerPort)]
+				if portMap.Scheme == "http" && !httpsOk && tlsEnabled {
+					processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
+						ContainerPort: portMap.ContainerPort,
+						HostPort:      443,
+						Name:          fmt.Sprintf("https-443-%d", portMap.ContainerPort),
+						Protocol:      PortmapProtocol_TCP,
+						Scheme:        "https",
+					})
+				}
+
+				if portMap.Scheme == "https" && !httpOk {
+					processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
+						ContainerPort: portMap.ContainerPort,
+						HostPort:      80,
+						Name:          fmt.Sprintf("http-80-%d", portMap.ContainerPort),
+						Protocol:      PortmapProtocol_TCP,
+						Scheme:        "http",
+					})
+				}
+			}
+
+			sort.Sort(NameSorter(processValues.Web.PortMaps))
+		} else if appJSON.Formation[processType].Service != nil && appJSON.Formation[processType].Service.Exposed {
+			processValues.Web = ProcessWeb{
+				Domains:  []ProcessDomains{},
+				PortMaps: []ProcessPortMap{},
+				TLS: ProcessTls{
+					Enabled: false,
+				},
+			}
+
+			processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
+				ContainerPort: 5000,
+				HostPort:      5000,
+				Name:          "http-5000-5000",
+				Protocol:      PortmapProtocol_TCP,
+				Scheme:        "http",
+			})
+		}
+
+		values.Processes[processType] = processValues
+
+		templateFiles := []string{"deployment", "keda-scaled-object"}
+		if processType == "web" {
+			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "compression-middleware", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
+		}
+		for _, templateName := range templateFiles {
+			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
+			if err != nil {
+				return cleanup(fmt.Errorf("Error reading %s template: %w", templateName, err))
+			}
+
+			filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
+			if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
+				return cleanup(fmt.Errorf("Error writing %s template: %w", templateName, err))
+			}
+
+			if os.Getenv("DOKKU_TRACE") == "1" {
+				common.CatFile(filename)
+			}
+		}
+	}
+
+	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
+		Namespace:     namespace,
+	})
+	if err != nil {
+		return cleanup(fmt.Errorf("Error listing cron jobs: %w", err))
+	}
+	for _, cronTask := range cronTasks {
+		suffix := ""
+		for _, cronJob := range cronJobs {
+			if cronJob.Annotations["dokku.com/cron-id"] == cronTask.ID {
+				if value, ok := cronJob.Annotations["dokku.com/job-suffix"]; ok {
+					suffix = value
+				}
+			}
+		}
+		if suffix == "" {
+			n := 5
+			b := make([]byte, n)
+			if _, err := rand.Read(b); err != nil {
+				return cleanup(fmt.Errorf("Error generating cron suffix: %w", err))
+			}
+			suffix = strings.ToLower(fmt.Sprintf("%X", b))
+		}
+
+		words, err := shellquote.Split(cronTask.Command)
+		if err != nil {
+			return cleanup(fmt.Errorf("Error parsing cron task command: %w", err))
+		}
+
+		processResources, err := getProcessResources(appName, cronTask.ID)
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting process resources: %w", err))
+		}
+
+		annotations, err := getAnnotations(appName, cronTask.ID)
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting process annotations: %w", err))
+		}
+
+		labels, err := getLabels(appName, cronTask.ID)
+		if err != nil {
+			return cleanup(fmt.Errorf("Error getting process labels: %w", err))
+		}
+
+		concurrencyPolicy := strings.ToUpper(cronTask.ConcurrencyPolicy)
+		switch concurrencyPolicy {
+		case "ALLOW":
+			concurrencyPolicy = "Allow"
+		case "FORBID":
+			concurrencyPolicy = "Forbid"
+		case "REPLACE":
+			concurrencyPolicy = "Replace"
+		default:
+			return cleanup(fmt.Errorf("Invalid concurrency_policy specified: %v", concurrencyPolicy))
+		}
+		processValues := ProcessValues{
+			Args:        words,
+			Annotations: annotations,
+			Cron: ProcessCron{
+				ID:                cronTask.ID,
+				Hash:              cronIDLabelValue(cronTask.ID),
+				Schedule:          cronTask.Schedule,
+				Suffix:            suffix,
+				Suspend:           cronTask.Maintenance,
+				ConcurrencyPolicy: ProcessCronConcurrencyPolicy(concurrencyPolicy),
+			},
+			Labels:      labels,
+			ProcessType: ProcessType_Cron,
+			Replicas:    1,
+			Resources:   processResources,
+			Volumes:     processVolumes,
+		}
+		values.Processes[cronTask.ID] = processValues
+	}
+
+	if len(cronTasks) > 0 {
+		b, err := templates.ReadFile("templates/chart/cron-job.yaml")
+		if err != nil {
+			return cleanup(fmt.Errorf("Error reading cron job template: %w", err))
+		}
+
+		cronFile := filepath.Join(chartDir, "templates", "cron-job.yaml")
+		if err := os.WriteFile(cronFile, b, os.FileMode(0644)); err != nil {
+			return cleanup(fmt.Errorf("Error writing cron job template: %w", err))
+		}
+
+		if os.Getenv("DOKKU_TRACE") == "1" {
+			common.CatFile(cronFile)
+		}
+	}
+
+	helpersBytes, err := templates.ReadFile("templates/chart/_helpers.tpl")
+	if err != nil {
+		return cleanup(fmt.Errorf("Error reading _helpers template: %w", err))
+	}
+
+	helpersFile := filepath.Join(chartDir, "templates", "_helpers.tpl")
+	if err := os.WriteFile(helpersFile, helpersBytes, os.FileMode(0644)); err != nil {
+		return cleanup(fmt.Errorf("Error writing _helpers template: %w", err))
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(helpersFile)
+	}
+
+	if err := writeYaml(WriteYamlInput{
+		Object: values,
+		Path:   filepath.Join(chartDir, "values.yaml"),
+	}); err != nil {
+		return cleanup(fmt.Errorf("Error writing chart: %w", err))
+	}
+
+	chartPath, err := filepath.Abs(chartDir)
+	if err != nil {
+		return cleanup(fmt.Errorf("Error getting chart path: %w", err))
+	}
+
+	kustomizeRootPath := ""
+	if hasKustomizeDirectory(appName) {
+		kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName)
+	}
+
+	result = BuildResult{
+		ChartDir:          chartDir,
+		ChartPath:         chartPath,
+		ReleaseName:       appName,
+		Namespace:         namespace,
+		Timeout:           timeoutDuration,
+		RollbackOnFailure: allowRollbacks,
+		KustomizeRootPath: kustomizeRootPath,
+	}
+	return result, nil
+}

plugins/scheduler-k3s/functions.go

@@ -1073,7 +1073,14 @@ func getIngressAnnotations(appName string, processType string) (map[string]strin
 	}
 
 	annotations := map[string]string{}
-	for _, newKey := range properties {
+	keys := make([]string, 0, len(properties))
+	for key := range properties {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		newKey := properties[key]
 		value := newKey.getter(appName)
 		if newKey.modifier != nil {
 			value = newKey.modifier(value)

plugins/scheduler-k3s/go.mod

@@ -4,6 +4,7 @@ go 1.26.2
 
 require (
 	github.com/Masterminds/semver/v3 v3.5.0
+	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
 	github.com/cert-manager/cert-manager v1.20.2
 	github.com/dokku/dokku/plugins/app-json v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
@@ -22,10 +23,13 @@ require (
 	github.com/gosimple/slug v1.15.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kedacore/keda/v2 v2.18.3
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
+	github.com/onsi/gomega v1.41.0
 	github.com/ryanuber/columnize v2.1.2+incompatible
 	github.com/spf13/pflag v1.0.10
 	github.com/traefik/traefik/v2 v2.11.46
 	golang.org/x/sync v0.20.0
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.21.0
 	k8s.io/api v0.35.4

plugins/scheduler-k3s/go.sum

@@ -24,6 +24,8 @@ github.com/alexellis/go-execute/v2 v2.2.1 h1:4Ye3jiCKQarstODOEmqDSRCqxMHLkC92Bhs
 github.com/alexellis/go-execute/v2 v2.2.1/go.mod h1:FMdRnUTiFAmYXcv23txrp3VYZfLo24nMpiIneWgKHTQ=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b h1:uUXgbcPDK3KpW29o4iy7GtuappbWT0l5NaMo9H9pJDw=
+github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -238,6 +240,8 @@ github.com/melbahja/goph v1.5.0 h1:RQUBpLvfg3i7fjfG8rTcSWyMjVRfdhwrrfQhjYee4dQ=
 github.com/melbahja/goph v1.5.0/go.mod h1:dDwo+44cmvfDLdiVpc6fJxexf5BA5yEDUeE5YgtuDO4=
 github.com/mgood/go-posix v0.0.0-20150821180505-948c005421f5 h1:AutzcJSqc7ROMqcjPKmxwLl//YrzDPb4tLSBlH3Irdc=
 github.com/mgood/go-posix v0.0.0-20150821180505-948c005421f5/go.mod h1:irVTeKOI2GrudEK6/mqR4dDlGH91lCZIZM34YKSKH7M=
+github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
+github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
 github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=

plugins/scheduler-k3s/internal/helmdiff/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

plugins/scheduler-k3s/internal/helmdiff/NOTICE.md

@@ -0,0 +1,28 @@
+# NOTICE
+
+Files in this directory are derived from
+[github.com/databus23/helm-diff](https://github.com/databus23/helm-diff),
+which is distributed under the Apache License 2.0. See `LICENSE` for the
+full text.
+
+Source files merged into this package:
+
+- `diff/diff.go`
+- `diff/report.go`
+- `manifest/parse.go`
+
+Modifications:
+
+- Merged the upstream `diff` and `manifest` packages into a single `helmdiff`
+  package; references to `manifest.MappingResult` are now bare `MappingResult`.
+- Removed three-way-merge support (the upstream `manifest/generate.go` and
+  `manifest/util.go` files), which imported Helm v4 and other dependencies not
+  used by Dokku.
+- Removed `manifest.ParseObject`, which depended on `k8s.io/apimachinery/pkg/runtime`.
+- Removed release-ownership tracking (`ManifestsOwnership`, `OwnershipDiff`,
+  `Releases`, `reIndexForRelease`).
+- Removed the simple, JSON, structured, dyff, and template output formatters and
+  the upstream `diff/structured.go` and `diff/constant.go` files. Only the
+  default unified-diff (`"diff"`) output format remains.
+- `setupReportFormat` is hard-wired to `setupDiffReport`.
+- `ReportEntry.Structured` and the `*Options.StructuredOutput()` method removed.

plugins/scheduler-k3s/internal/helmdiff/diff.go

@@ -0,0 +1,520 @@
+// Derived from github.com/databus23/helm-diff (Apache License 2.0).
+// See LICENSE and NOTICE.md in this directory.
+
+package helmdiff
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/aryann/difflib"
+	"github.com/mgutz/ansi"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+// Options controls the behavior of Manifests.
+type Options struct {
+	OutputFormat              string
+	OutputContext             int
+	StripTrailingCR           bool
+	ShowSecrets               bool
+	ShowSecretsDecoded        bool
+	SuppressedKinds           []string
+	FindRenames               float32
+	SuppressedOutputLineRegex []string
+}
+
+const kindSecret = "Secret"
+
+// Manifests compares two indexed manifest maps and writes a unified diff to to.
+// It returns true if any changes were detected.
+func Manifests(oldIndex, newIndex map[string]*MappingResult, options *Options, to io.Writer) bool {
+	seenAnyChanges, report, err := generateReport(oldIndex, newIndex, options)
+	if err != nil {
+		panic(err)
+	}
+
+	report.print(to)
+	report.clean()
+	return seenAnyChanges
+}
+
+// ManifestReport is like Manifests but returns the constructed Report instead
+// of writing it. Useful for tests.
+func ManifestReport(oldIndex, newIndex map[string]*MappingResult, options *Options) (*Report, error) {
+	_, report, err := generateReport(oldIndex, newIndex, options)
+	return report, err
+}
+
+func generateReport(oldIndex, newIndex map[string]*MappingResult, options *Options) (bool, *Report, error) {
+	report := Report{findRenames: options.FindRenames}
+	report.setupReportFormat(options.OutputFormat)
+	var possiblyRemoved []string
+
+	for _, key := range sortedKeys(oldIndex) {
+		oldContent := oldIndex[key]
+
+		if newContent, ok := newIndex[key]; ok {
+			doDiff(&report, key, oldContent, newContent, options)
+		} else {
+			possiblyRemoved = append(possiblyRemoved, key)
+		}
+	}
+
+	var possiblyAdded []string
+	for _, key := range sortedKeys(newIndex) {
+		if _, ok := oldIndex[key]; !ok {
+			possiblyAdded = append(possiblyAdded, key)
+		}
+	}
+
+	removed, added := contentSearch(&report, possiblyRemoved, oldIndex, possiblyAdded, newIndex, options)
+
+	for _, key := range removed {
+		oldContent := oldIndex[key]
+		if oldContent.ResourcePolicy != "keep" {
+			doDiff(&report, key, oldContent, nil, options)
+		}
+	}
+
+	for _, key := range added {
+		newContent := newIndex[key]
+		doDiff(&report, key, nil, newContent, options)
+	}
+
+	seenAnyChanges := len(report.Entries) > 0
+
+	report, err := doSuppress(report, options.SuppressedOutputLineRegex)
+
+	return seenAnyChanges, &report, err
+}
+
+func doSuppress(report Report, suppressedOutputLineRegex []string) (Report, error) {
+	if len(report.Entries) == 0 || len(suppressedOutputLineRegex) == 0 {
+		return report, nil
+	}
+
+	filteredReport := Report{
+		findRenames: report.findRenames,
+	}
+	filteredReport.format = report.format
+	filteredReport.Entries = []ReportEntry{}
+
+	var suppressOutputRegexes []*regexp.Regexp
+
+	for _, suppressOutputRegex := range suppressedOutputLineRegex {
+		regex, err := regexp.Compile(suppressOutputRegex)
+		if err != nil {
+			return Report{}, err
+		}
+
+		suppressOutputRegexes = append(suppressOutputRegexes, regex)
+	}
+
+	for _, entry := range report.Entries {
+		var diffs []difflib.DiffRecord
+
+	DIFFS:
+		for _, diff := range entry.Diffs {
+			for _, suppressOutputRegex := range suppressOutputRegexes {
+				if suppressOutputRegex.MatchString(diff.Payload) {
+					continue DIFFS
+				}
+			}
+
+			diffs = append(diffs, diff)
+		}
+
+		containsDiff := false
+
+		for _, diff := range diffs {
+			if diff.Delta.String() != " " {
+				containsDiff = true
+				break
+			}
+		}
+
+		diffRecords := []difflib.DiffRecord{}
+		switch {
+		case containsDiff:
+			diffRecords = diffs
+		case entry.ChangeType == "MODIFY":
+			entry.ChangeType = "MODIFY_SUPPRESSED"
+		}
+
+		filteredReport.addEntry(entry.Key, entry.SuppressedKinds, entry.Kind, entry.Context, diffRecords, entry.ChangeType)
+	}
+
+	return filteredReport, nil
+}
+
+func actualChanges(diff []difflib.DiffRecord) int {
+	changes := 0
+	for _, record := range diff {
+		if record.Delta != difflib.Common {
+			changes++
+		}
+	}
+	return changes
+}
+
+const (
+	renameDetectionMinLengthRatio float32 = 0.1
+	renameDetectionMaxLengthRatio float32 = 10.0
+)
+
+func contentSearch(report *Report, possiblyRemoved []string, oldIndex map[string]*MappingResult, possiblyAdded []string, newIndex map[string]*MappingResult, options *Options) ([]string, []string) {
+	if options.FindRenames <= 0 {
+		return possiblyRemoved, possiblyAdded
+	}
+
+	var removed []string
+
+	for _, removedKey := range possiblyRemoved {
+		oldContent := oldIndex[removedKey]
+		var smallestKey string
+		var smallestFraction float32 = math.MaxFloat32
+		for _, addedKey := range possiblyAdded {
+			newContent := newIndex[addedKey]
+			if oldContent.Kind != newContent.Kind {
+				continue
+			}
+
+			oldLen := len(oldContent.Content)
+			newLen := len(newContent.Content)
+			if oldLen == 0 || newLen == 0 {
+				continue
+			}
+			if oldContent.Kind != kindSecret {
+				ratio := float32(oldLen) / float32(newLen)
+				if ratio < renameDetectionMinLengthRatio || ratio > renameDetectionMaxLengthRatio {
+					continue
+				}
+			}
+
+			switch {
+			case options.ShowSecretsDecoded:
+				decodeSecrets(oldContent, newContent)
+			case !options.ShowSecrets:
+				redactSecrets(oldContent, newContent)
+			}
+
+			diff := diffMappingResults(oldContent, newContent, options.StripTrailingCR)
+			delta := actualChanges(diff)
+			if delta == 0 || len(diff) == 0 {
+				continue
+			}
+			fraction := float32(delta) / float32(len(diff))
+			if fraction > 0 && fraction < smallestFraction {
+				smallestKey = addedKey
+				smallestFraction = fraction
+			}
+		}
+
+		if smallestFraction < options.FindRenames {
+			index := sort.SearchStrings(possiblyAdded, smallestKey)
+			possiblyAdded = append(possiblyAdded[:index], possiblyAdded[index+1:]...)
+			newContent := newIndex[smallestKey]
+			doDiff(report, removedKey, oldContent, newContent, options)
+		} else {
+			removed = append(removed, removedKey)
+		}
+	}
+
+	return removed, possiblyAdded
+}
+
+func doDiff(report *Report, key string, oldContent *MappingResult, newContent *MappingResult, options *Options) {
+	if oldContent != nil && newContent != nil && oldContent.Content == newContent.Content {
+		return
+	}
+	switch {
+	case options.ShowSecretsDecoded:
+		decodeSecrets(oldContent, newContent)
+	case !options.ShowSecrets:
+		redactSecrets(oldContent, newContent)
+	}
+
+	var changeType string
+	var subjectKind string
+	var diffs []difflib.DiffRecord
+	switch {
+	case oldContent == nil:
+		changeType = "ADD"
+		if newContent != nil {
+			subjectKind = newContent.Kind
+			emptyMapping := &MappingResult{}
+			diffs = diffMappingResults(emptyMapping, newContent, options.StripTrailingCR)
+		}
+	case newContent == nil:
+		changeType = "REMOVE"
+		subjectKind = oldContent.Kind
+		emptyMapping := &MappingResult{}
+		diffs = diffMappingResults(oldContent, emptyMapping, options.StripTrailingCR)
+	default:
+		changeType = "MODIFY"
+		subjectKind = oldContent.Kind
+		diffs = diffMappingResults(oldContent, newContent, options.StripTrailingCR)
+		if actualChanges(diffs) == 0 {
+			return
+		}
+	}
+
+	report.addEntry(key, options.SuppressedKinds, subjectKind, options.OutputContext, diffs, changeType)
+}
+
+func preHandleSecrets(old, new *MappingResult) (v1.Secret, v1.Secret, error, error) {
+	var oldSecretDecodeErr, newSecretDecodeErr error
+	var oldSecret, newSecret v1.Secret
+	if old != nil {
+		oldSecretDecodeErr = yaml.NewYAMLToJSONDecoder(bytes.NewBufferString(old.Content)).Decode(&oldSecret)
+		if oldSecretDecodeErr != nil {
+			old.Content = fmt.Sprintf("Error parsing old secret: %s", oldSecretDecodeErr)
+		} else {
+			if len(oldSecret.StringData) > 0 && oldSecret.Data == nil {
+				oldSecret.Data = make(map[string][]byte, len(oldSecret.StringData))
+			}
+			for k, v := range oldSecret.StringData {
+				oldSecret.Data[k] = []byte(v)
+			}
+		}
+	}
+	if new != nil {
+		newSecretDecodeErr = yaml.NewYAMLToJSONDecoder(bytes.NewBufferString(new.Content)).Decode(&newSecret)
+		if newSecretDecodeErr != nil {
+			new.Content = fmt.Sprintf("Error parsing new secret: %s", newSecretDecodeErr)
+		} else {
+			if len(newSecret.StringData) > 0 && newSecret.Data == nil {
+				newSecret.Data = make(map[string][]byte, len(newSecret.StringData))
+			}
+			for k, v := range newSecret.StringData {
+				newSecret.Data[k] = []byte(v)
+			}
+		}
+	}
+	return oldSecret, newSecret, oldSecretDecodeErr, newSecretDecodeErr
+}
+
+// redactSecrets replaces Secret data values with placeholders so the actual
+// secret bytes never appear in diff output.
+func redactSecrets(old, new *MappingResult) {
+	if (old != nil && old.Kind != kindSecret) || (new != nil && new.Kind != kindSecret) {
+		return
+	}
+	serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme)
+
+	oldSecret, newSecret, oldSecretDecodeErr, newSecretDecodeErr := preHandleSecrets(old, new)
+
+	if old != nil && oldSecretDecodeErr == nil {
+		oldSecret.StringData = make(map[string]string, len(oldSecret.Data))
+		for k, v := range oldSecret.Data {
+			if new != nil && bytes.Equal(v, newSecret.Data[k]) {
+				oldSecret.StringData[k] = fmt.Sprintf("REDACTED # (%d bytes)", len(v))
+			} else {
+				oldSecret.StringData[k] = fmt.Sprintf("-------- # (%d bytes)", len(v))
+			}
+		}
+	}
+	if new != nil && newSecretDecodeErr == nil {
+		newSecret.StringData = make(map[string]string, len(newSecret.Data))
+		for k, v := range newSecret.Data {
+			if old != nil && bytes.Equal(v, oldSecret.Data[k]) {
+				newSecret.StringData[k] = fmt.Sprintf("REDACTED # (%d bytes)", len(v))
+			} else {
+				newSecret.StringData[k] = fmt.Sprintf("++++++++ # (%d bytes)", len(v))
+			}
+		}
+	}
+
+	if old != nil && oldSecretDecodeErr == nil {
+		oldSecretBuf := bytes.NewBuffer(nil)
+		oldSecret.Data = nil
+		if err := serializer.Encode(&oldSecret, oldSecretBuf); err != nil {
+			new.Content = fmt.Sprintf("Error encoding new secret: %s", err)
+		}
+		old.Content = getComment(old.Content) + strings.Replace(strings.Replace(oldSecretBuf.String(), "stringData", "data", 1), "  creationTimestamp: null\n", "", 1)
+		oldSecretBuf.Reset()
+	}
+	if new != nil && newSecretDecodeErr == nil {
+		newSecretBuf := bytes.NewBuffer(nil)
+		newSecret.Data = nil
+		if err := serializer.Encode(&newSecret, newSecretBuf); err != nil {
+			new.Content = fmt.Sprintf("Error encoding new secret: %s", err)
+		}
+		new.Content = getComment(new.Content) + strings.Replace(strings.Replace(newSecretBuf.String(), "stringData", "data", 1), "  creationTimestamp: null\n", "", 1)
+		newSecretBuf.Reset()
+	}
+}
+
+// decodeSecrets renders Secret data values base64-decoded in diff output.
+func decodeSecrets(old, new *MappingResult) {
+	if (old != nil && old.Kind != kindSecret) || (new != nil && new.Kind != kindSecret) {
+		return
+	}
+	serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme)
+
+	oldSecret, newSecret, oldSecretDecodeErr, newSecretDecodeErr := preHandleSecrets(old, new)
+
+	if old != nil && oldSecretDecodeErr == nil {
+		oldSecret.StringData = make(map[string]string, len(oldSecret.Data))
+		for k, v := range oldSecret.Data {
+			oldSecret.StringData[k] = string(v)
+		}
+	}
+	if new != nil && newSecretDecodeErr == nil {
+		newSecret.StringData = make(map[string]string, len(newSecret.Data))
+		for k, v := range newSecret.Data {
+			newSecret.StringData[k] = string(v)
+		}
+	}
+
+	if old != nil && oldSecretDecodeErr == nil {
+		oldSecretBuf := bytes.NewBuffer(nil)
+		oldSecret.Data = nil
+		if err := serializer.Encode(&oldSecret, oldSecretBuf); err != nil {
+			new.Content = fmt.Sprintf("Error encoding new secret: %s", err)
+		}
+		old.Content = getComment(old.Content) + strings.Replace(oldSecretBuf.String(), "  creationTimestamp: null\n", "", 1)
+		oldSecretBuf.Reset()
+	}
+	if new != nil && newSecretDecodeErr == nil {
+		newSecretBuf := bytes.NewBuffer(nil)
+		newSecret.Data = nil
+		if err := serializer.Encode(&newSecret, newSecretBuf); err != nil {
+			new.Content = fmt.Sprintf("Error encoding new secret: %s", err)
+		}
+		new.Content = getComment(new.Content) + strings.Replace(newSecretBuf.String(), "  creationTimestamp: null\n", "", 1)
+		newSecretBuf.Reset()
+	}
+}
+
+// getComment returns the first line of a string if it is a comment. This
+// preserves the leading "# Source: ..." lines from Helm-rendered manifests.
+func getComment(s string) string {
+	i := strings.Index(s, "\n")
+	if i < 0 || !strings.HasPrefix(s, "#") {
+		return ""
+	}
+	return s[:i+1]
+}
+
+func diffMappingResults(oldContent *MappingResult, newContent *MappingResult, stripTrailingCR bool) []difflib.DiffRecord {
+	return diffStrings(oldContent.Content, newContent.Content, stripTrailingCR)
+}
+
+func diffStrings(before, after string, stripTrailingCR bool) []difflib.DiffRecord {
+	return difflib.Diff(split(before, stripTrailingCR), split(after, stripTrailingCR))
+}
+
+func split(value string, stripTrailingCR bool) []string {
+	const sep = "\n"
+	split := strings.Split(value, sep)
+	if !stripTrailingCR {
+		return split
+	}
+	var stripped []string
+	for _, s := range split {
+		stripped = append(stripped, strings.TrimSuffix(s, "\r"))
+	}
+	return stripped
+}
+
+func printDiffRecords(suppressedKinds []string, kind string, context int, diffs []difflib.DiffRecord, to io.Writer) {
+	for _, ckind := range suppressedKinds {
+		if ckind == kind {
+			str := fmt.Sprintf("+ Changes suppressed on sensitive content of type %s\n", kind)
+			_, _ = fmt.Fprint(to, ansi.Color(str, "yellow"))
+			return
+		}
+	}
+
+	if context >= 0 {
+		distances := calculateDistances(diffs)
+		omitting := false
+		for i, diff := range diffs {
+			if distances[i] > context {
+				if !omitting {
+					_, _ = fmt.Fprintln(to, "...")
+					omitting = true
+				}
+			} else {
+				omitting = false
+				printDiffRecord(diff, to)
+			}
+		}
+	} else {
+		for _, diff := range diffs {
+			printDiffRecord(diff, to)
+		}
+	}
+}
+
+func printDiffRecord(diff difflib.DiffRecord, to io.Writer) {
+	text := diff.Payload
+
+	switch diff.Delta {
+	case difflib.RightOnly:
+		_, _ = fmt.Fprintf(to, "%s\n", ansi.Color("+ "+text, "green"))
+	case difflib.LeftOnly:
+		_, _ = fmt.Fprintf(to, "%s\n", ansi.Color("- "+text, "red"))
+	case difflib.Common:
+		if text == "" {
+			_, _ = fmt.Fprintln(to)
+		} else {
+			_, _ = fmt.Fprintf(to, "%s\n", "  "+text)
+		}
+	}
+}
+
+// calculateDistances returns the distance of every diff line to the closest change.
+func calculateDistances(diffs []difflib.DiffRecord) map[int]int {
+	distances := map[int]int{}
+
+	change := -1
+	for i, diff := range diffs {
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		distance := math.MaxInt32
+		if change != -1 {
+			distance = i - change
+		}
+		distances[i] = distance
+	}
+
+	change = -1
+	for i := len(diffs) - 1; i >= 0; i-- {
+		diff := diffs[i]
+		if diff.Delta != difflib.Common {
+			change = i
+		}
+		if change != -1 {
+			distance := change - i
+			if distance < distances[i] {
+				distances[i] = distance
+			}
+		}
+	}
+
+	return distances
+}
+
+func sortedKeys(manifests map[string]*MappingResult) []string {
+	var keys []string
+
+	for key := range manifests {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	return keys
+}

plugins/scheduler-k3s/internal/helmdiff/diff_test.go

@@ -0,0 +1,118 @@
+package helmdiff
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+const sampleDeployment = `# Source: app/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web
+  namespace: default
+spec:
+  replicas: 1
+`
+
+const sampleDeploymentChanged = `# Source: app/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web
+  namespace: default
+spec:
+  replicas: 3
+`
+
+const sampleSecret = `# Source: app/templates/secret.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: creds
+  namespace: default
+type: Opaque
+data:
+  password: c2VjcmV0
+`
+
+const sampleSecretChanged = `# Source: app/templates/secret.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: creds
+  namespace: default
+type: Opaque
+data:
+  password: bmV3LXNlY3JldA==
+`
+
+func runDiff(t *testing.T, oldManifest, newManifest string, opts *Options) string {
+	t.Helper()
+	current := Parse([]byte(oldManifest), "default", true)
+	proposed := Parse([]byte(newManifest), "default", true)
+	var buf bytes.Buffer
+	Manifests(current, proposed, opts, &buf)
+	return buf.String()
+}
+
+func TestManifestsNoChange(t *testing.T) {
+	out := runDiff(t, sampleDeployment, sampleDeployment, &Options{OutputContext: 3})
+	if out != "" {
+		t.Fatalf("expected empty diff for identical manifests, got:\n%s", out)
+	}
+}
+
+func TestManifestsSingleFieldChange(t *testing.T) {
+	out := runDiff(t, sampleDeployment, sampleDeploymentChanged, &Options{OutputContext: 3})
+	if !strings.Contains(out, "has changed") {
+		t.Fatalf("expected MODIFY header in output, got:\n%s", out)
+	}
+	if !strings.Contains(out, "replicas: 1") {
+		t.Fatalf("expected removed line for replicas: 1, got:\n%s", out)
+	}
+	if !strings.Contains(out, "replicas: 3") {
+		t.Fatalf("expected added line for replicas: 3, got:\n%s", out)
+	}
+}
+
+func TestManifestsAdditiveWhenOldEmpty(t *testing.T) {
+	out := runDiff(t, "", sampleDeployment, &Options{OutputContext: 3})
+	if !strings.Contains(out, "has been added") {
+		t.Fatalf("expected ADD header in output, got:\n%s", out)
+	}
+	if !strings.Contains(out, "kind: Deployment") {
+		t.Fatalf("expected added line for kind: Deployment, got:\n%s", out)
+	}
+}
+
+func TestManifestsSecretRedactedByDefault(t *testing.T) {
+	out := runDiff(t, sampleSecret, sampleSecretChanged, &Options{OutputContext: -1})
+	if strings.Contains(out, "c2VjcmV0") || strings.Contains(out, "bmV3LXNlY3JldA==") {
+		t.Fatalf("expected raw base64 secret values to be redacted, got:\n%s", out)
+	}
+	if !strings.Contains(out, "# (") || !strings.Contains(out, "bytes)") {
+		t.Fatalf("expected byte-count placeholders in redacted output, got:\n%s", out)
+	}
+}
+
+func TestManifestsSecretShownWithShowSecrets(t *testing.T) {
+	out := runDiff(t, sampleSecret, sampleSecretChanged, &Options{OutputContext: -1, ShowSecrets: true})
+	if !strings.Contains(out, "c2VjcmV0") {
+		t.Fatalf("expected old base64 value to appear with ShowSecrets, got:\n%s", out)
+	}
+	if !strings.Contains(out, "bmV3LXNlY3JldA==") {
+		t.Fatalf("expected new base64 value to appear with ShowSecrets, got:\n%s", out)
+	}
+}
+
+func TestManifestsSecretDecodedWithShowSecretsDecoded(t *testing.T) {
+	out := runDiff(t, sampleSecret, sampleSecretChanged, &Options{OutputContext: -1, ShowSecretsDecoded: true})
+	if !strings.Contains(out, "password: secret") {
+		t.Fatalf("expected old plaintext value with ShowSecretsDecoded, got:\n%s", out)
+	}
+	if !strings.Contains(out, "password: new-secret") {
+		t.Fatalf("expected new plaintext value with ShowSecretsDecoded, got:\n%s", out)
+	}
+}

plugins/scheduler-k3s/internal/helmdiff/parse.go

@@ -0,0 +1,184 @@
+// Derived from github.com/databus23/helm-diff (Apache License 2.0).
+// See LICENSE and NOTICE.md in this directory.
+
+package helmdiff
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	hookAnnotation           = "helm.sh/hook"
+	resourcePolicyAnnotation = "helm.sh/resource-policy"
+)
+
+var yamlSeparator = []byte("\n---\n")
+
+// MappingResult stores a single Kubernetes object rendered from a Helm chart.
+type MappingResult struct {
+	Name           string
+	Kind           string
+	Content        string
+	ResourcePolicy string
+}
+
+type metadata struct {
+	APIVersion string `yaml:"apiVersion"`
+	Kind       string
+	Metadata   struct {
+		Namespace   string
+		Name        string
+		Annotations map[string]string
+	}
+}
+
+func (m metadata) String() string {
+	apiBase := m.APIVersion
+	sp := strings.Split(apiBase, "/")
+	if len(sp) > 1 {
+		apiBase = strings.Join(sp[:len(sp)-1], "/")
+	}
+	name := m.Metadata.Name
+	if a := m.Metadata.Annotations; a != nil {
+		if baseName, ok := a["helm-diff/base-name"]; ok {
+			name = baseName
+		}
+	}
+	return fmt.Sprintf("%s, %s, %s (%s)", m.Metadata.Namespace, name, m.Kind, apiBase)
+}
+
+func scanYamlSpecs(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+	if i := bytes.Index(data, yamlSeparator); i >= 0 {
+		return i + len(yamlSeparator), data[0:i], nil
+	}
+	if atEOF {
+		return len(data), data, nil
+	}
+	return 0, nil, nil
+}
+
+// Parse parses manifest bytes into a map of MappingResults keyed by
+// "<namespace>, <name>, <kind> (<apiVersion>)".
+func Parse(manifest []byte, defaultNamespace string, normalizeManifests bool, excludedHooks ...string) map[string]*MappingResult {
+	scanner := bufio.NewScanner(io.MultiReader(strings.NewReader("\n"), bytes.NewReader(manifest)))
+	scanner.Split(scanYamlSpecs)
+	scanner.Buffer(make([]byte, bufio.MaxScanTokenSize), 10485760)
+
+	result := make(map[string]*MappingResult)
+
+	for scanner.Scan() {
+		content := bytes.TrimSpace(scanner.Bytes())
+		if len(content) == 0 {
+			continue
+		}
+
+		parsed, err := parseContent(content, defaultNamespace, normalizeManifests, excludedHooks...)
+		if err != nil {
+			log.Fatalf("%v", err)
+		}
+
+		for _, p := range parsed {
+			name := p.Name
+
+			if _, ok := result[name]; ok {
+				log.Printf("Error: Found duplicate key %#v in manifest", name)
+			} else {
+				result[name] = p
+			}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatalf("Error reading input: %s", err)
+	}
+	return result
+}
+
+func parseContent(content []byte, defaultNamespace string, normalizeManifests bool, excludedHooks ...string) ([]*MappingResult, error) {
+	var parsedMetadata metadata
+	if err := yaml.Unmarshal(content, &parsedMetadata); err != nil {
+		log.Fatalf("YAML unmarshal error: %s\nCan't unmarshal %s", err, content)
+	}
+
+	if parsedMetadata.APIVersion == "" && parsedMetadata.Kind == "" {
+		return nil, nil
+	}
+
+	if strings.HasSuffix(parsedMetadata.Kind, "List") {
+		type ListV1 struct {
+			Items []yaml.MapSlice `yaml:"items"`
+		}
+
+		var list ListV1
+
+		if err := yaml.Unmarshal(content, &list); err != nil {
+			log.Fatalf("YAML unmarshal error: %s\nCan't unmarshal %s", err, content)
+		}
+
+		var result []*MappingResult
+
+		for _, item := range list.Items {
+			subcontent, err := yaml.Marshal(item)
+			if err != nil {
+				log.Printf("YAML marshal error: %s\nCan't marshal %v", err, item)
+			}
+
+			subs, err := parseContent(subcontent, defaultNamespace, normalizeManifests, excludedHooks...)
+			if err != nil {
+				return nil, fmt.Errorf("parsing YAML list item: %w", err)
+			}
+
+			result = append(result, subs...)
+		}
+
+		return result, nil
+	}
+
+	if normalizeManifests {
+		var object map[interface{}]interface{}
+		if err := yaml.Unmarshal(content, &object); err != nil {
+			log.Fatalf("YAML unmarshal error: %s\nCan't unmarshal %s", err, content)
+		}
+		normalizedContent, err := yaml.Marshal(object)
+		if err != nil {
+			log.Fatalf("YAML marshal error: %s\nCan't marshal %v", err, object)
+		}
+		content = normalizedContent
+	}
+
+	if isHook(parsedMetadata, excludedHooks...) {
+		return nil, nil
+	}
+
+	if parsedMetadata.Metadata.Namespace == "" {
+		parsedMetadata.Metadata.Namespace = defaultNamespace
+	}
+
+	name := parsedMetadata.String()
+	return []*MappingResult{
+		{
+			Name:           name,
+			Kind:           parsedMetadata.Kind,
+			Content:        string(content),
+			ResourcePolicy: parsedMetadata.Metadata.Annotations[resourcePolicyAnnotation],
+		},
+	}, nil
+}
+
+func isHook(metadata metadata, hooks ...string) bool {
+	for _, hook := range hooks {
+		if metadata.Metadata.Annotations[hookAnnotation] == hook {
+			return true
+		}
+	}
+	return false
+}

plugins/scheduler-k3s/internal/helmdiff/report.go

@@ -0,0 +1,93 @@
+// Derived from github.com/databus23/helm-diff (Apache License 2.0).
+// See LICENSE and NOTICE.md in this directory.
+
+package helmdiff
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/aryann/difflib"
+	"github.com/mgutz/ansi"
+)
+
+// Report stores diff report data and formatting state.
+type Report struct {
+	format      ReportFormat
+	Entries     []ReportEntry
+	mode        string
+	findRenames float32
+}
+
+// ReportEntry stores changes for a single Kubernetes object.
+type ReportEntry struct {
+	Key             string
+	SuppressedKinds []string
+	Kind            string
+	Context         int
+	Diffs           []difflib.DiffRecord
+	ChangeType      string
+}
+
+// ReportFormat holds the output renderer and change-type styles.
+type ReportFormat struct {
+	output       func(r *Report, to io.Writer)
+	changestyles map[string]ChangeStyle
+}
+
+// ChangeStyle controls how a single change type is rendered.
+type ChangeStyle struct {
+	color   string
+	message string
+}
+
+// setupReportFormat configures the report's output renderer.
+// Only the default unified-diff format is supported.
+func (r *Report) setupReportFormat(format string) {
+	r.mode = format
+	setupDiffReport(r)
+}
+
+// addEntry appends a single diff entry to the report.
+func (r *Report) addEntry(key string, suppressedKinds []string, kind string, context int, diffs []difflib.DiffRecord, changeType string) {
+	entry := ReportEntry{
+		key,
+		suppressedKinds,
+		kind,
+		context,
+		diffs,
+		changeType,
+	}
+	r.Entries = append(r.Entries, entry)
+}
+
+// print writes all entries via the configured output renderer.
+func (r *Report) print(to io.Writer) {
+	r.format.output(r, to)
+}
+
+// clean resets the entries slice (used by tests).
+func (r *Report) clean() {
+	r.Entries = nil
+}
+
+func setupDiffReport(r *Report) {
+	r.format.output = printDiffReport
+	r.format.changestyles = make(map[string]ChangeStyle)
+	r.format.changestyles["ADD"] = ChangeStyle{color: "green", message: "has been added:"}
+	r.format.changestyles["REMOVE"] = ChangeStyle{color: "red", message: "has been removed:"}
+	r.format.changestyles["MODIFY"] = ChangeStyle{color: "yellow", message: "has changed:"}
+	r.format.changestyles["MODIFY_SUPPRESSED"] = ChangeStyle{color: "blue+h", message: "has changed, but diff is empty after suppression."}
+}
+
+func printDiffReport(r *Report, to io.Writer) {
+	for _, entry := range r.Entries {
+		_, _ = fmt.Fprintf(
+			to,
+			ansi.Color("%s %s", r.format.changestyles[entry.ChangeType].color)+"\n",
+			entry.Key,
+			r.format.changestyles[entry.ChangeType].message,
+		)
+		printDiffRecords(entry.SuppressedKinds, entry.Kind, entry.Context, entry.Diffs, to)
+	}
+}

plugins/scheduler-k3s/preview.go

@@ -0,0 +1,192 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/dokku/dokku/plugins/common"
+	"github.com/dokku/dokku/plugins/scheduler-k3s/internal/helmdiff"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/release"
+	"helm.sh/helm/v3/pkg/storage/driver"
+)
+
+// CommandPreview prints a unified diff between the manifests currently stored
+// in the live Helm release for an app and the manifests that the next deploy
+// would roll out. Only the main app helm release is compared; the auxiliary
+// config-secret, image-pull-secret, and TLS-secret releases are out of scope.
+//
+// diffContext controls the number of unchanged lines shown around each change;
+// pass -1 to print the full resource.
+func CommandPreview(appName string, diffContext int, showSecrets bool, showSecretsDecoded bool) error {
+	if err := common.VerifyAppName(appName); err != nil {
+		return err
+	}
+
+	scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "")
+	globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local")
+	if scheduler == "" {
+		scheduler = globalScheduler
+	}
+	if scheduler != "k3s" {
+		return fmt.Errorf("App %s is not configured to use the k3s scheduler", appName)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
+	namespace := getComputedNamespace(appName)
+	helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
+	if err != nil {
+		return fmt.Errorf("Error creating helm agent: %w", err)
+	}
+
+	currentRelease, err := fetchCurrentRelease(helmAgent, appName)
+	if err != nil {
+		return err
+	}
+
+	buildOpts := BuildOptions{AllowMissingImage: true}
+	if currentRelease != nil {
+		buildOpts.OverrideDeploymentID = deploymentIDFromRelease(currentRelease)
+	}
+
+	chartResult, err := BuildAppChart(ctx, appName, "", buildOpts)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(chartResult.ChartDir)
+
+	chartRequested, err := loadChartForPreview(chartResult.ChartPath)
+	if err != nil {
+		return err
+	}
+
+	var currentManifest, proposedManifest string
+	if currentRelease != nil {
+		currentManifest = currentRelease.Manifest
+		proposedManifest, err = renderUpgradeDryRun(ctx, helmAgent, chartResult, chartRequested)
+	} else {
+		proposedManifest, err = renderInstallDryRun(ctx, helmAgent, chartResult, chartRequested)
+	}
+	if err != nil {
+		return err
+	}
+
+	options := &helmdiff.Options{
+		OutputFormat:       "diff",
+		OutputContext:      diffContext,
+		ShowSecrets:        showSecrets,
+		ShowSecretsDecoded: showSecretsDecoded,
+	}
+
+	current := helmdiff.Parse([]byte(currentManifest), chartResult.Namespace, true)
+	proposed := helmdiff.Parse([]byte(proposedManifest), chartResult.Namespace, true)
+	helmdiff.Manifests(current, proposed, options, os.Stdout)
+	return nil
+}
+
+func loadChartForPreview(chartPath string) (*chart.Chart, error) {
+	settings := cli.New()
+	opts := action.ChartPathOptions{}
+	located, err := opts.LocateChart(chartPath, settings)
+	if err != nil {
+		return nil, fmt.Errorf("Error locating chart: %w", err)
+	}
+	chartRequested, err := loader.Load(located)
+	if err != nil {
+		return nil, fmt.Errorf("Error loading chart: %w", err)
+	}
+	return chartRequested, nil
+}
+
+func fetchCurrentRelease(helmAgent *HelmAgent, releaseName string) (*release.Release, error) {
+	getAction := action.NewGet(helmAgent.Configuration)
+	rel, err := getAction.Run(releaseName)
+	if err != nil {
+		if errors.Is(err, driver.ErrReleaseNotFound) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("Error fetching current release: %w", err)
+	}
+	return rel, nil
+}
+
+// deploymentIDFromRelease extracts the deployment id encoded in a release's
+// chart version. BuildAppChart writes the chart version as "0.0.<id>"
+// (chart.go); the parsed id lets CommandPreview pin the previewed manifests
+// to the currently-deployed value so the `app.kubernetes.io/version`
+// annotation does not churn in the diff. Returns 0 when the version does not
+// match the expected shape - the caller then falls back to the timestamp.
+func deploymentIDFromRelease(rel *release.Release) int64 {
+	if rel == nil || rel.Chart == nil || rel.Chart.Metadata == nil {
+		return 0
+	}
+	const prefix = "0.0."
+	v := rel.Chart.Metadata.Version
+	if !strings.HasPrefix(v, prefix) {
+		return 0
+	}
+	id, err := strconv.ParseInt(strings.TrimPrefix(v, prefix), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return id
+}
+
+func renderUpgradeDryRun(ctx context.Context, helmAgent *HelmAgent, chartResult BuildResult, chartRequested *chart.Chart) (string, error) {
+	kustomizeRenderer := KustomizeRenderer{
+		ReleaseName:       chartResult.ReleaseName,
+		KustomizeRootPath: chartResult.KustomizeRootPath,
+	}
+
+	client := action.NewUpgrade(helmAgent.Configuration)
+	client.DryRun = true
+	client.DryRunOption = "client"
+	client.Namespace = chartResult.Namespace
+	client.PostRenderer = &kustomizeRenderer
+
+	rel, err := client.RunWithContext(ctx, chartResult.ReleaseName, chartRequested, map[string]interface{}{})
+	if err != nil {
+		return "", fmt.Errorf("Error rendering upgrade dry-run: %w", err)
+	}
+	return rel.Manifest, nil
+}
+
+func renderInstallDryRun(ctx context.Context, helmAgent *HelmAgent, chartResult BuildResult, chartRequested *chart.Chart) (string, error) {
+	kustomizeRenderer := KustomizeRenderer{
+		ReleaseName:       chartResult.ReleaseName,
+		KustomizeRootPath: chartResult.KustomizeRootPath,
+	}
+
+	client := action.NewInstall(helmAgent.Configuration)
+	client.DryRun = true
+	client.ClientOnly = true
+	client.Namespace = chartResult.Namespace
+	client.ReleaseName = chartResult.ReleaseName
+	client.PostRenderer = &kustomizeRenderer
+
+	rel, err := client.RunWithContext(ctx, chartRequested, map[string]interface{}{})
+	if err != nil {
+		return "", fmt.Errorf("Error rendering install dry-run: %w", err)
+	}
+	return rel.Manifest, nil
+}

plugins/scheduler-k3s/src/commands/commands.go

@@ -29,6 +29,7 @@ Additional commands:`
     scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster
     scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
     scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
+    scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
     scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster
     scheduler-k3s:profiles:list [--format json|stdout], Lists all node profiles in the k3s cluster
     scheduler-k3s:profiles:remove <profile>, Removes a node profile from the k3s cluster

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -158,6 +158,14 @@ func main() {
 			appName = "--global"
 		}
 		err = scheduler_k3s.CommandLabelsReport(appName, *format, *processType, *resourceType, infoFlag)
+	case "preview":
+		args := flag.NewFlagSet("scheduler-k3s:preview", flag.ExitOnError)
+		context := args.Int("context", 3, "--context: number of unchanged lines of context around each change (-1 for full output)")
+		showSecrets := args.Bool("show-secrets", false, "--show-secrets: do not redact secret values in the output")
+		showSecretsDecoded := args.Bool("show-secrets-decoded", false, "--show-secrets-decoded: render secret values base64-decoded")
+		args.Parse(os.Args[2:])
+		appName := args.Arg(0)
+		err = scheduler_k3s.CommandPreview(appName, *context, *showSecrets, *showSecretsDecoded)
 	case "profiles:add":
 		args := flag.NewFlagSet("scheduler-k3s:profiles:add", flag.ExitOnError)
 		role := args.String("role", "worker", "role: [ server | worker ]")

plugins/scheduler-k3s/triggers.go

@@ -3,7 +3,6 @@ package scheduler_k3s
 import (
 	"bytes"
 	"context"
-	"crypto/rand"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -11,20 +10,16 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
-	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/dokku/dokku/plugins/config"
 	"github.com/dokku/dokku/plugins/cron"
 	nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts"
 	"github.com/dokku/dokku/plugins/registry"
 	"github.com/fatih/color"
-	"github.com/gosimple/slug"
 	"github.com/kballard/go-shellquote"
 	"github.com/ryanuber/columnize"
 	corev1 "k8s.io/api/core/v1"
@@ -463,18 +458,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	if scheduler != "k3s" {
 		return nil
 	}
-	results, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
-		Trigger: "ps-current-scale",
-		Args:    []string{appName},
-	})
-	if err != nil {
-		return err
-	}
-
-	processes, err := common.ParseScaleOutput(results.StdoutBytes())
-	if err != nil {
-		return err
-	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
@@ -493,82 +476,27 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error creating kubernetes namespace for deployment: %w", err)
 	}
 
-	image, err := common.GetDeployingAppImageName(appName, imageTag, "")
+	chartResult, err := BuildAppChart(ctx, appName, imageTag, BuildOptions{})
 	if err != nil {
-		return fmt.Errorf("Error getting deploying app image name: %w", err)
-	}
-
-	deployTimeout := getComputedDeployTimeout(appName)
-	if _, err := strconv.Atoi(deployTimeout); err == nil {
-		deployTimeout = fmt.Sprintf("%ss", deployTimeout)
-	}
-
-	deployRollback := getComputedRollbackOnFailure(appName)
-	allowRollbacks, err := strconv.ParseBool(deployRollback)
-	if err != nil {
-		return fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err)
-	}
-
-	imageSourceType := "dockerfile"
-	if common.IsImageCnbBased(image) {
-		imageSourceType = "pack"
-	} else if common.IsImageHerokuishBased(image, appName) {
-		imageSourceType = "herokuish"
+		return err
 	}
+	defer os.RemoveAll(chartResult.ChartDir)
 
 	env, err := config.LoadMergedAppEnv(appName)
 	if err != nil {
 		return fmt.Errorf("Error loading environment for deployment: %w", err)
 	}
 
-	// Check for imported TLS certificate first
-	importedCertExists := false
-	if HasImportedTLSCert(appName) {
-		exists, err := TLSSecretExists(ctx, appName)
-		if err == nil && exists {
-			importedCertExists = true
-		}
-	}
-
-	// Determine TLS configuration
-	tlsEnabled := false
-	issuerName := ""
-	useImportedCert := false
-
-	if importedCertExists {
-		tlsEnabled = true
-		useImportedCert = true
-	} else {
-		server := getComputedLetsencryptServer(appName)
-		letsencryptEmailStag := getGlobalLetsencryptEmailStag()
-		letsencryptEmailProd := getGlobalLetsencryptEmailProd()
-
-		switch server {
-		case "prod", "production":
-			issuerName = "letsencrypt-prod"
-			tlsEnabled = letsencryptEmailProd != ""
-		case "stag", "staging":
-			issuerName = "letsencrypt-stag"
-			tlsEnabled = letsencryptEmailStag != ""
-		case "false":
-			issuerName = ""
-			tlsEnabled = false
-		default:
-			return fmt.Errorf("Invalid letsencrypt server config: %s", server)
-		}
-	}
-
-	chartDir, err := os.MkdirTemp("", "dokku-chart-")
+	globalAnnotations, err := getGlobalAnnotations(appName)
 	if err != nil {
-		return fmt.Errorf("Error creating chart directory: %w", err)
-	}
-	defer os.RemoveAll(chartDir)
-
-	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
-		return fmt.Errorf("Error creating chart templates directory: %w", err)
+		return fmt.Errorf("Error getting global annotations: %w", err)
+	}
+
+	globalLabels, err := getGlobalLabel(appName)
+	if err != nil {
+		return fmt.Errorf("Error getting global labels: %w", err)
 	}
 
-	deploymentId := time.Now().Unix()
 	imagePullSecrets := getComputedImagePullSecrets(appName)
 	dokkuManagedPullSecret := false
 	var dokkuPullSecretBytes []byte
@@ -586,484 +514,18 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		}
 	}
 
-	globalTemplateFiles := []string{"service-account"}
-	for _, templateName := range globalTemplateFiles {
-		b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
-		if err != nil {
-			return fmt.Errorf("Error reading %s template: %w", templateName, err)
-		}
-
-		filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
-		err = os.WriteFile(filename, b, os.FileMode(0644))
-		if err != nil {
-			return fmt.Errorf("Error writing %s template: %w", templateName, err)
-		}
-
-		if os.Getenv("DOKKU_TRACE") == "1" {
-			common.CatFile(filename)
-		}
-	}
-
-	portMaps, err := getPortMaps(appName)
-	if err != nil {
-		return fmt.Errorf("Error getting port mappings for deployment: %w", err)
-	}
-
-	primaryPort := int32(5000)
-	primaryServicePort := int32(80)
-	for _, portMap := range portMaps {
-		primaryPort = portMap.ContainerPort
-		primaryServicePort = portMap.HostPort
-		if primaryPort != 0 {
-			break
-		}
-	}
-
-	appJSON, err := appjson.GetAppJSON(appName)
-	if err != nil {
-		return fmt.Errorf("Error getting app.json for deployment: %w", err)
-	}
-
-	workingDir := common.GetWorkingDir(appName, image)
-
-	cronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
-	if err != nil {
-		return fmt.Errorf("Error fetching cron tasks: %w", err)
-	}
-
-	domains := []string{}
-	if _, ok := processes["web"]; ok {
-		_, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
-			Trigger:     "domains-vhost-enabled",
-			Args:        []string{appName},
-			StreamStdio: true,
-		})
-		if err == nil {
-			results, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
-				Trigger: "domains-list",
-				Args:    []string{appName},
-			})
-			if err != nil {
-				return fmt.Errorf("Error getting domains for deployment: %w", err)
-			}
-
-			for _, domain := range strings.Split(results.StdoutContents(), "\n") {
-				domain = strings.TrimSpace(domain)
-				if domain != "" {
-					domains = append(domains, domain)
-				}
-			}
-		}
-	}
-
-	chart := &Chart{
-		ApiVersion: "v2",
-		AppVersion: "1.0.0",
-		Name:       appName,
-		Icon:       "https://dokku.com/assets/dokku-logo.svg",
-		Version:    fmt.Sprintf("0.0.%d", deploymentId),
-	}
-
-	err = writeYaml(WriteYamlInput{
-		Object: chart,
-		Path:   filepath.Join(chartDir, "Chart.yaml"),
-	})
-	if err != nil {
-		return fmt.Errorf("Error writing chart: %w", err)
-	}
-
-	globalAnnotations, err := getGlobalAnnotations(appName)
-	if err != nil {
-		return fmt.Errorf("Error getting global annotations: %w", err)
-	}
-
-	globalLabels, err := getGlobalLabel(appName)
-	if err != nil {
-		return fmt.Errorf("Error getting global labels: %w", err)
-	}
-
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	if err := clientset.Ping(); err != nil {
-		return fmt.Errorf("kubernetes api not available: %w", err)
-	}
-
-	kedaValues, err := getKedaValues(ctx, clientset, appName)
-	if err != nil {
-		return fmt.Errorf("Error getting keda values: %w", err)
-	}
-
-	securityContext, err := getSecurityContext(appName, "deploy")
-	if err != nil {
-		return fmt.Errorf("Error getting security context: %w", err)
-	}
-
-	values := &AppValues{
-		Global: GlobalValues{
-			Annotations:  globalAnnotations,
-			AppName:      appName,
-			DeploymentID: fmt.Sprint(deploymentId),
-			Keda:         kedaValues,
-			Image: GlobalImage{
-				ImagePullSecrets: imagePullSecrets,
-				Name:             image,
-				Type:             imageSourceType,
-				WorkingDir:       workingDir,
-			},
-			Labels:    globalLabels,
-			Namespace: namespace,
-			Network: GlobalNetwork{
-				IngressClass:       getComputedIngressClass(),
-				PrimaryPort:        primaryPort,
-				PrimaryServicePort: primaryServicePort,
-			},
-			SecurityContext: securityContext,
-		},
-		Processes: map[string]ProcessValues{},
-	}
-
-	if len(kedaValues.Authentications) > 0 {
-		templateFiles := []string{"keda-secret", "keda-trigger-authentication"}
-		for _, templateName := range templateFiles {
-			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
-			if err != nil {
-				return fmt.Errorf("Error reading %s template: %w", templateName, err)
-			}
-
-			filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
-			err = os.WriteFile(filename, b, os.FileMode(0644))
-			if err != nil {
-				return fmt.Errorf("Error writing %s template: %w", templateName, err)
-			}
-
-			if os.Getenv("DOKKU_TRACE") == "1" {
-				common.CatFile(filename)
-			}
-		}
-	}
-
-	processVolumes := []ProcessVolume{}
-	if shmSize := getComputedShmSize(appName); shmSize != "" {
-		processVolumes = append(processVolumes, ProcessVolume{
-			Name:      "shmem",
-			MountPath: "/dev/shm",
-			EmptyDir: &ProcessVolumeEmptyDir{
-				Medium:    "Memory",
-				SizeLimit: shmSize,
-			},
-		})
-	}
-
-	deployMountPairs, err := LoadAppMounts(appName, "deploy")
-	if err != nil {
-		return fmt.Errorf("error loading storage app mounts: %w", err)
-	}
-	deployVolumes, err := ToProcessVolumes(deployMountPairs)
-	if err != nil {
-		return err
-	}
-	processVolumes = append(processVolumes, deployVolumes...)
-
-	for processType, processCount := range processes {
-		// todo: implement deployment annotations
-		// todo: implement pod annotations
-
-		healthchecks, ok := appJSON.Healthchecks[processType]
-		if !ok {
-			healthchecks = []appjson.Healthcheck{}
-		}
-		processHealthchecks := getProcessHealtchecks(healthchecks, primaryPort)
-
-		startCommand, err := getStartCommand(StartCommandInput{
-			AppName:         appName,
-			ProcessType:     processType,
-			ImageSourceType: imageSourceType,
-			Port:            primaryPort,
-			Env:             env.Map(),
-		})
-		if err != nil {
-			return fmt.Errorf("Error getting start command for deployment: %w", err)
-		}
-		args := startCommand.Command
-
-		processResources, err := getProcessResources(appName, processType)
-		if err != nil {
-			return fmt.Errorf("Error getting process resources: %w", err)
-		}
-
-		annotations, err := getAnnotations(appName, processType)
-		if err != nil {
-			return fmt.Errorf("Error getting process annotations: %w", err)
-		}
-
-		labels, err := getLabels(appName, processType)
-		if err != nil {
-			return fmt.Errorf("Error getting process labels: %w", err)
-		}
-
-		autoscaling, err := getAutoscaling(GetAutoscalingInput{
-			AppName:     appName,
-			ProcessType: processType,
-			Replicas:    int(processCount),
-			KedaValues:  kedaValues,
-		})
-		if err != nil {
-			return fmt.Errorf("Error getting autoscaling: %w", err)
-		}
-
-		processValues := ProcessValues{
-			Annotations:  annotations,
-			Autoscaling:  autoscaling,
-			Args:         args,
-			Healthchecks: processHealthchecks,
-			Labels:       labels,
-			ProcessType:  ProcessType_Worker,
-			Replicas:     int32(processCount),
-			Resources:    processResources,
-			Volumes:      processVolumes,
-		}
-
-		if processType == "web" {
-			sort.Strings(domains)
-			domainValues := []ProcessDomains{}
-			for _, domain := range domains {
-				domainValues = append(domainValues, ProcessDomains{
-					Name: domain,
-					Slug: slug.Make(domain),
-				})
-			}
-
-			processValues.Web = ProcessWeb{
-				Domains:  domainValues,
-				PortMaps: []ProcessPortMap{},
-				TLS: ProcessTls{
-					Enabled:         tlsEnabled,
-					IssuerName:      issuerName,
-					UseImportedCert: useImportedCert,
-				},
-			}
-
-			processValues.ProcessType = ProcessType_Web
-			for _, portMap := range portMaps {
-				protocol := PortmapProtocol_TCP
-				if portMap.Scheme == "udp" {
-					protocol = PortmapProtocol_UDP
-				}
-
-				processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
-					ContainerPort: portMap.ContainerPort,
-					HostPort:      portMap.HostPort,
-					Name:          portMap.String(),
-					Protocol:      protocol,
-					Scheme:        portMap.Scheme,
-				})
-			}
-
-			for _, portMap := range processValues.Web.PortMaps {
-				_, httpOk := portMaps[fmt.Sprintf("http-80-%d", portMap.ContainerPort)]
-				_, httpsOk := portMaps[fmt.Sprintf("https-443-%d", portMap.ContainerPort)]
-				if portMap.Scheme == "http" && !httpsOk && tlsEnabled {
-					processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
-						ContainerPort: portMap.ContainerPort,
-						HostPort:      443,
-						Name:          fmt.Sprintf("https-443-%d", portMap.ContainerPort),
-						Protocol:      PortmapProtocol_TCP,
-						Scheme:        "https",
-					})
-				}
-
-				if portMap.Scheme == "https" && !httpOk {
-					processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
-						ContainerPort: portMap.ContainerPort,
-						HostPort:      80,
-						Name:          fmt.Sprintf("http-80-%d", portMap.ContainerPort),
-						Protocol:      PortmapProtocol_TCP,
-						Scheme:        "http",
-					})
-				}
-			}
-
-			sort.Sort(NameSorter(processValues.Web.PortMaps))
-		} else if appJSON.Formation[processType].Service != nil && appJSON.Formation[processType].Service.Exposed {
-			processValues.Web = ProcessWeb{
-				Domains:  []ProcessDomains{},
-				PortMaps: []ProcessPortMap{},
-				TLS: ProcessTls{
-					Enabled: false,
-				},
-			}
-
-			processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
-				ContainerPort: 5000,
-				HostPort:      5000,
-				Name:          "http-5000-5000",
-				Protocol:      PortmapProtocol_TCP,
-				Scheme:        "http",
-			})
-		}
-
-		values.Processes[processType] = processValues
-
-		templateFiles := []string{"deployment", "keda-scaled-object"}
-		if processType == "web" {
-			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "compression-middleware", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
-		}
-		for _, templateName := range templateFiles {
-			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
-			if err != nil {
-				return fmt.Errorf("Error reading %s template: %w", templateName, err)
-			}
-
-			filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
-			err = os.WriteFile(filename, b, os.FileMode(0644))
-			if err != nil {
-				return fmt.Errorf("Error writing %s template: %w", templateName, err)
-			}
-
-			if os.Getenv("DOKKU_TRACE") == "1" {
-				common.CatFile(filename)
-			}
-		}
-	}
-
-	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
-		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
-		Namespace:     namespace,
-	})
-	if err != nil {
-		return fmt.Errorf("Error listing cron jobs: %w", err)
-	}
-	for _, cronTask := range cronTasks {
-		// todo: implement deployment annotations
-		// todo: implement pod annotations
-		suffix := ""
-		for _, cronJob := range cronJobs {
-			if cronJob.Annotations["dokku.com/cron-id"] == cronTask.ID {
-				if value, ok := cronJob.Annotations["dokku.com/job-suffix"]; ok {
-					suffix = value
-				}
-			}
-		}
-		if suffix == "" {
-			n := 5
-			b := make([]byte, n)
-			if _, err := rand.Read(b); err != nil {
-				panic(err)
-			}
-			suffix = strings.ToLower(fmt.Sprintf("%X", b))
-		}
-
-		words, err := shellquote.Split(cronTask.Command)
-		if err != nil {
-			return fmt.Errorf("Error parsing cron task command: %w", err)
-		}
-
-		processResources, err := getProcessResources(appName, cronTask.ID)
-		if err != nil {
-			return fmt.Errorf("Error getting process resources: %w", err)
-		}
-
-		annotations, err := getAnnotations(appName, cronTask.ID)
-		if err != nil {
-			return fmt.Errorf("Error getting process annotations: %w", err)
-		}
-
-		labels, err := getLabels(appName, cronTask.ID)
-		if err != nil {
-			return fmt.Errorf("Error getting process labels: %w", err)
-		}
-
-		concurrencyPolicy := strings.ToUpper(cronTask.ConcurrencyPolicy)
-		switch concurrencyPolicy {
-		case "ALLOW":
-			concurrencyPolicy = "Allow"
-		case "FORBID":
-			concurrencyPolicy = "Forbid"
-		case "REPLACE":
-			concurrencyPolicy = "Replace"
-		default:
-			return fmt.Errorf("Invalid concurrency_policy specified: %v", concurrencyPolicy)
-		}
-		processValues := ProcessValues{
-			Args:        words,
-			Annotations: annotations,
-			Cron: ProcessCron{
-				ID:                cronTask.ID,
-				Hash:              cronIDLabelValue(cronTask.ID),
-				Schedule:          cronTask.Schedule,
-				Suffix:            suffix,
-				Suspend:           cronTask.Maintenance,
-				ConcurrencyPolicy: ProcessCronConcurrencyPolicy(concurrencyPolicy),
-			},
-			Labels:      labels,
-			ProcessType: ProcessType_Cron,
-			Replicas:    1,
-			Resources:   processResources,
-			Volumes:     processVolumes,
-		}
-		values.Processes[cronTask.ID] = processValues
-	}
-
-	if len(cronTasks) > 0 {
-		b, err := templates.ReadFile("templates/chart/cron-job.yaml")
-		if err != nil {
-			return fmt.Errorf("Error reading cron job template: %w", err)
-		}
-
-		cronFile := filepath.Join(chartDir, "templates", "cron-job.yaml")
-		err = os.WriteFile(cronFile, b, os.FileMode(0644))
-		if err != nil {
-			return fmt.Errorf("Error writing cron job template: %w", err)
-		}
-
-		if os.Getenv("DOKKU_TRACE") == "1" {
-			common.CatFile(cronFile)
-		}
-	}
-
-	b, err := templates.ReadFile("templates/chart/_helpers.tpl")
-	if err != nil {
-		return fmt.Errorf("Error reading _helpers template: %w", err)
-	}
-
-	helpersFile := filepath.Join(chartDir, "templates", "_helpers.tpl")
-	err = os.WriteFile(helpersFile, b, os.FileMode(0644))
-	if err != nil {
-		return fmt.Errorf("Error writing _helpers template: %w", err)
-	}
-
-	if os.Getenv("DOKKU_TRACE") == "1" {
-		common.CatFile(helpersFile)
-	}
-
-	err = writeYaml(WriteYamlInput{
-		Object: values,
-		Path:   filepath.Join(chartDir, "values.yaml"),
-	})
-	if err != nil {
-		return fmt.Errorf("Error writing chart: %w", err)
-	}
-
-	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	helmAgent, err := NewHelmAgent(chartResult.Namespace, DeployLogPrinter)
 	if err != nil {
 		return fmt.Errorf("Error creating helm agent: %w", err)
 	}
 
-	chartPath, err := filepath.Abs(chartDir)
-	if err != nil {
-		return fmt.Errorf("Error getting chart path: %w", err)
-	}
-
-	timeoutDuration, err := time.ParseDuration(deployTimeout)
-	if err != nil {
-		return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
-	}
-
 	ingresses, err := clientset.ListIngresses(ctx, ListIngressesInput{
-		Namespace:     namespace,
+		Namespace:     chartResult.Namespace,
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/instance=%s-web", appName),
 	})
 	if err != nil {
@@ -1086,7 +548,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		common.LogVerboseQuiet(fmt.Sprintf("Removing non-matching ingress resource: %s", ingressName))
 		err := clientset.DeleteIngress(ctx, DeleteIngressInput{
 			Name:      ingressName,
-			Namespace: namespace,
+			Namespace: chartResult.Namespace,
 		})
 
 		if err != nil {
@@ -1122,23 +584,18 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	if imagePullSecrets != "" {
 		keepImagePullSecrets = []string{imagePullSecrets}
 	}
-	if err := pruneStaleImagePullSecretsFromDeployments(ctx, clientset, namespace, appName, keepImagePullSecrets); err != nil {
+	if err := pruneStaleImagePullSecretsFromDeployments(ctx, clientset, chartResult.Namespace, appName, keepImagePullSecrets); err != nil {
 		return fmt.Errorf("Error pruning stale imagePullSecrets entries: %w", err)
 	}
 
-	kustomizeRootPath := ""
-	if hasKustomizeDirectory(appName) {
-		kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName)
-	}
-
 	common.LogInfo2(fmt.Sprintf("Installing %s", appName))
 	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
-		ChartPath:         chartPath,
-		KustomizeRootPath: kustomizeRootPath,
-		Namespace:         namespace,
-		ReleaseName:       appName,
-		RollbackOnFailure: allowRollbacks,
-		Timeout:           timeoutDuration,
+		ChartPath:         chartResult.ChartPath,
+		KustomizeRootPath: chartResult.KustomizeRootPath,
+		Namespace:         chartResult.Namespace,
+		ReleaseName:       chartResult.ReleaseName,
+		RollbackOnFailure: chartResult.RollbackOnFailure,
+		Timeout:           chartResult.Timeout,
 		Wait:              true,
 	})
 	if err != nil {

tests/unit/scheduler-k3s-preview.bats

@@ -0,0 +1,91 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s:preview) clean redeploy shows no diff" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:preview $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+}
+
+@test "(scheduler-k3s:preview) resource limit change appears in diff" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku resource:limit --process-type web --memory 256 $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:preview $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "has changed"
+  assert_output_contains "256Mi"
+}
+
+@test "(scheduler-k3s:preview) undeployed app shows additive diff" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:preview $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "has been added" -1
+}