Merge pull request #8549 from dokku/8526-improve-scheduler-k3s-support-for-alternative-builders-cnb-etc

Route CNB images through launcher on scheduler-k3s

plugins/scheduler-k3s/cron_job_template_test.go

@@ -0,0 +1,149 @@
+package scheduler_k3s
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+)
+
+// TestCronIDLabelValue asserts the hashed cron ID fits inside Kubernetes' 63
+// byte label cap and is deterministic. Without this the label can exceed the
+// cap and the Kubernetes API server rejects the manifest.
+func TestCronIDLabelValue(t *testing.T) {
+	cases := []string{
+		"short",
+		"app===echo hello-from-cron===5 5 5 5 5",
+		strings.Repeat("a-very-long-cron-id-that-would-far-exceed-the-label-cap-", 10),
+	}
+	for _, in := range cases {
+		out := cronIDLabelValue(in)
+		if len(out) > 63 {
+			t.Errorf("cronIDLabelValue(%q) = %q (len %d); must be <= 63", in, out, len(out))
+		}
+		if out != cronIDLabelValue(in) {
+			t.Errorf("cronIDLabelValue(%q) is not deterministic", in)
+		}
+	}
+}
+
+// TestCronJobTemplateQuotesAllDigitSuffix asserts that the rendered cron-job
+// manifest produces string-typed annotation values even when the suffix and
+// cron-id consist entirely of digits. Without `| quote` in the template, YAML
+// would coerce these to numbers and the manifest would be rejected by the
+// Kubernetes API server.
+func TestCronJobTemplateQuotesAllDigitSuffix(t *testing.T) {
+	chartDir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n")
+	if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil {
+		t.Fatalf("write Chart.yaml: %v", err)
+	}
+
+	cronJobTpl, err := templates.ReadFile("templates/chart/cron-job.yaml")
+	if err != nil {
+		t.Fatalf("read cron-job template: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(chartDir, "templates", "cron-job.yaml"), cronJobTpl, 0o644); err != nil {
+		t.Fatalf("write cron-job template: %v", err)
+	}
+
+	helpersTpl, err := templates.ReadFile("templates/chart/_helpers.tpl")
+	if err != nil {
+		t.Fatalf("read _helpers: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(chartDir, "templates", "_helpers.tpl"), helpersTpl, 0o644); err != nil {
+		t.Fatalf("write _helpers: %v", err)
+	}
+
+	loaded, err := loader.Load(chartDir)
+	if err != nil {
+		t.Fatalf("load chart: %v", err)
+	}
+
+	values := map[string]interface{}{
+		"global": map[string]interface{}{
+			"app_name":      "myapp",
+			"deployment_id": "1",
+			"namespace":     "myapp",
+			"image": map[string]interface{}{
+				"name": "myapp:latest",
+				"type": "dockerfile",
+			},
+		},
+		"processes": map[string]interface{}{
+			"cron-id-123": map[string]interface{}{
+				"args": []interface{}{"echo", "hello"},
+				"cron": map[string]interface{}{
+					"id":                 "1234567890",
+					"hash":               "abc123def",
+					"schedule":           "5 5 5 5 5",
+					"suffix":             "1234567890",
+					"suspend":            false,
+					"concurrency_policy": "Allow",
+				},
+			},
+		},
+	}
+
+	renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil)
+	if err != nil {
+		t.Fatalf("ToRenderValues: %v", err)
+	}
+
+	rendered, err := engine.Render(loaded, renderValues)
+	if err != nil {
+		t.Fatalf("render: %v", err)
+	}
+
+	var manifest string
+	for name, content := range rendered {
+		if filepath.Base(name) == "cron-job.yaml" {
+			manifest = content
+			break
+		}
+	}
+	if manifest == "" {
+		t.Fatalf("cron-job.yaml not rendered; got: %v", rendered)
+	}
+
+	decoder := yaml.NewDecoder(strings.NewReader(manifest))
+	for {
+		var doc map[string]interface{}
+		if err := decoder.Decode(&doc); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			t.Fatalf("yaml decode failed (would also fail in Kubernetes API): %v\nrendered:\n%s", err, manifest)
+		}
+		if doc == nil {
+			continue
+		}
+		metadata, ok := doc["metadata"].(map[string]interface{})
+		if !ok {
+			continue
+		}
+		annotations, _ := metadata["annotations"].(map[string]interface{})
+		for key, value := range annotations {
+			if _, isString := value.(string); !isString {
+				t.Errorf("annotation %q has non-string value %v (type %T); helm template must apply | quote", key, value, value)
+			}
+		}
+		labels, _ := metadata["labels"].(map[string]interface{})
+		for key, value := range labels {
+			if _, isString := value.(string); !isString {
+				t.Errorf("label %q has non-string value %v (type %T); helm template must apply | quote", key, value, value)
+			}
+		}
+	}
+}

plugins/scheduler-k3s/functions.go

@@ -3,6 +3,7 @@ package scheduler_k3s
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -23,6 +24,7 @@ import (
 	nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts"
 	resty "github.com/go-resty/resty/v2"
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
+	"github.com/multiformats/go-base36"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/yaml.v3"
 	"helm.sh/helm/v3/pkg/strvals"
@@ -134,6 +136,23 @@ type WaitForPodBySelectorRunningInput struct {
 	Waiter func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
 }
 
+type WaitForPodBySelectorCompletedInput struct {
+	// Clientset is the kubernetes clientset
+	Clientset KubernetesClient
+
+	// Namespace is the namespace to search in
+	Namespace string
+
+	// LabelSelector is the label selector to search for
+	LabelSelector string
+
+	// PodName is the pod name to search for
+	PodName string
+
+	// Timeout is the timeout in seconds to wait for the pod to reach a terminal phase
+	Timeout float64
+}
+
 type WaitForPodToExistInput struct {
 	Clientset     KubernetesClient
 	Namespace     string
@@ -1981,6 +2000,52 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp
 	}
 }
 
+// waitForPodBySelectorCompleted polls the first pod matching the input
+// selector until it reaches a terminal phase (PodSucceeded / PodFailed) or
+// the timeout elapses. Used after a Follow=true StreamLogs returns, to
+// handle the kubelet -> apiserver phase propagation delay for short-lived
+// run pods.
+func waitForPodBySelectorCompleted(ctx context.Context, input WaitForPodBySelectorCompletedInput) (v1.Pod, error) {
+	pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
+		Clientset:     input.Clientset,
+		LabelSelector: input.LabelSelector,
+		Namespace:     input.Namespace,
+		PodName:       input.PodName,
+		RetryCount:    3,
+	})
+	if err != nil {
+		return v1.Pod{}, fmt.Errorf("Error waiting for pod to exist: %w", err)
+	}
+
+	if len(pods) == 0 {
+		return v1.Pod{}, fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.LabelSelector)
+	}
+
+	podName := pods[0].Name
+	if input.PodName != "" {
+		podName = input.PodName
+	}
+
+	timeout := time.Duration(input.Timeout * float64(time.Second))
+	var pod v1.Pod
+	err = wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) {
+		p, err := input.Clientset.GetPod(ctx, GetPodInput{
+			Name:      podName,
+			Namespace: input.Namespace,
+		})
+		if err != nil {
+			return false, err
+		}
+		pod = p
+		switch p.Status.Phase {
+		case v1.PodSucceeded, v1.PodFailed:
+			return true, nil
+		}
+		return false, nil
+	})
+	return pod, err
+}
+
 // kubernetesNodeToNode converts a kubernetes node to a Node
 func kubernetesNodeToNode(node v1.Node) Node {
 	roles := []string{}
@@ -2018,6 +2083,14 @@ func kubernetesNodeToNode(node v1.Node) Node {
 	}
 }
 
+// cronIDLabelValue returns a Kubernetes-label-safe hash of a cron ID.
+// The raw cron ID can exceed the 63-byte label cap, so we keep it as an
+// annotation and use this short hash for selectors.
+func cronIDLabelValue(cronID string) string {
+	sum := sha256.Sum256([]byte(cronID))
+	return base36.EncodeToStringLc(sum[:16])
+}
+
 // parseMemoryQuantity parses a string into a valid memory quantity
 func parseMemoryQuantity(input string) (string, error) {
 	if _, err := strconv.ParseInt(input, 10, 64); err == nil {

plugins/scheduler-k3s/go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/gosimple/slug v1.15.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kedacore/keda/v2 v2.18.3
+	github.com/multiformats/go-base36 v0.2.0
 	github.com/ryanuber/columnize v2.1.2+incompatible
 	github.com/spf13/pflag v1.0.10
 	github.com/traefik/traefik/v2 v2.11.46
@@ -115,7 +116,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
-	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect

plugins/scheduler-k3s/template.go

@@ -305,6 +305,7 @@ const (
 
 type ProcessCron struct {
 	ID                string                       `yaml:"id"`
+	Hash              string                       `yaml:"hash"`
 	Schedule          string                       `yaml:"schedule"`
 	Suffix            string                       `yaml:"suffix"`
 	Suspend           bool                         `yaml:"suspend"`

plugins/scheduler-k3s/templates/chart/cron-job.yaml

@@ -9,18 +9,18 @@ kind: CronJob
 metadata:
   annotations:
     app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
-    dokku.com/builder-type: {{ $.Values.global.image.type }}
+    dokku.com/builder-type: {{ $.Values.global.image.type | quote }}
-    dokku.com/cron-id: {{ $config.cron.id }}
+    dokku.com/cron-id: {{ $config.cron.id | quote }}
-    dokku.com/job-suffix: {{ $config.cron.suffix }}
+    dokku.com/job-suffix: {{ $config.cron.suffix | quote }}
     dokku.com/managed: "true"
-    kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
+    kubectl.kubernetes.io/default-container: {{ printf "%s-cron" $.Values.global.app_name | quote }}
     {{ include "print.annotations" (dict "config" $.Values.global "key" "cronjob") | indent 4 }}
     {{ include "print.annotations" (dict "config" $config "key" "cronjob") | indent 4 }}
   labels:
-    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
+    app.kubernetes.io/instance: {{ printf "%s-cron-%s" $.Values.global.app_name $config.cron.suffix | quote }}
     app.kubernetes.io/name: cron
-    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }}
-    dokku.com/cron-id: {{ $config.cron.id }}
+    dokku.com/cron-hash: {{ $config.cron.hash | quote }}
     {{ include "print.labels" (dict "config" $.Values.global "key" "cronjob") | indent 4 }}
     {{ include "print.labels" (dict "config" $config "key" "cronjob") | indent 4 }}
   name: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
@@ -32,18 +32,18 @@ spec:
     metadata:
       annotations:
         app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
-        dokku.com/builder-type: {{ $.Values.global.image.type }}
+        dokku.com/builder-type: {{ $.Values.global.image.type | quote }}
-        dokku.com/cron-id: {{ $config.cron.id }}
+        dokku.com/cron-id: {{ $config.cron.id | quote }}
-        dokku.com/job-suffix: {{ $config.cron.suffix }}
+        dokku.com/job-suffix: {{ $config.cron.suffix | quote }}
         dokku.com/managed: "true"
-        kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
+        kubectl.kubernetes.io/default-container: {{ printf "%s-cron" $.Values.global.app_name | quote }}
         {{ include "print.annotations" (dict "config" $.Values.global "key" "job") | indent 8 }}
         {{ include "print.annotations" (dict "config" $config "key" "job") | indent 8 }}
       labels:
-        app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
+        app.kubernetes.io/instance: {{ printf "%s-cron-%s" $.Values.global.app_name $config.cron.suffix | quote }}
         app.kubernetes.io/name: cron
-        app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+        app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }}
-        dokku.com/cron-id: {{ $config.cron.id }}
+        dokku.com/cron-hash: {{ $config.cron.hash | quote }}
         {{ include "print.labels" (dict "config" $.Values.global "key" "job") | indent 8 }}
         {{ include "print.labels" (dict "config" $config "key" "job") | indent 8 }}
     spec:
@@ -55,18 +55,18 @@ spec:
         metadata:
           annotations:
             app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
-            dokku.com/builder-type: {{ $.Values.global.image.type }}
+            dokku.com/builder-type: {{ $.Values.global.image.type | quote }}
-            dokku.com/cron-id: {{ $config.cron.id }}
+            dokku.com/cron-id: {{ $config.cron.id | quote }}
-            dokku.com/job-suffix: {{ $config.cron.suffix }}
+            dokku.com/job-suffix: {{ $config.cron.suffix | quote }}
             dokku.com/managed: "true"
-            kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
+            kubectl.kubernetes.io/default-container: {{ printf "%s-cron" $.Values.global.app_name | quote }}
             {{ include "print.annotations" (dict "config" $.Values.global "key" "pod") | indent 12 }}
             {{ include "print.annotations" (dict "config" $config "key" "pod") | indent 12 }}
           labels:
-            app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
+            app.kubernetes.io/instance: {{ printf "%s-cron-%s" $.Values.global.app_name $config.cron.suffix | quote }}
             app.kubernetes.io/name: cron
-            app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+            app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }}
-            dokku.com/cron-id: {{ $config.cron.id }}
+            dokku.com/cron-hash: {{ $config.cron.hash | quote }}
             {{ include "print.labels" (dict "config" $.Values.global "key" "pod") | indent 12 }}
             {{ include "print.labels" (dict "config" $config "key" "pod") | indent 12 }}
         spec:
@@ -75,6 +75,10 @@ spec:
             {{- range $config.args }}
             - {{ . }}
             {{- end }}
+            {{- if eq $.Values.global.image.type "pack" }}
+            command:
+            - launcher
+            {{- end }}
             envFrom:
             - secretRef:
                 name: config-{{ $.Values.global.app_name }}

plugins/scheduler-k3s/templates/chart/deployment.yaml

@@ -16,15 +16,15 @@ kind: Deployment
 metadata:
   annotations:
     app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
-    dokku.com/builder-type: {{ $.Values.global.image.type }}
+    dokku.com/builder-type: {{ $.Values.global.image.type | quote }}
     dokku.com/managed: "true"
-    kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
+    kubectl.kubernetes.io/default-container: {{ printf "%s-%s" $.Values.global.app_name $processName | quote }}
     {{ include "print.annotations" (dict "config" $.Values.global "key" "deployment") | indent 4 }}
     {{ include "print.annotations" (dict "config" $config "key" "deployment") | indent 4 }}
   labels:
-    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/instance: {{ printf "%s-%s" $.Values.global.app_name $processName | quote }}
-    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/name: {{ $processName | quote }}
-    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }}
     {{ include "print.labels" (dict "config" $.Values.global "key" "deployment") | indent 4 }}
     {{ include "print.labels" (dict "config" $config "key" "deployment") | indent 4 }}
   name: {{ $.Values.global.app_name }}-{{ $processName }}
@@ -63,6 +63,10 @@ spec:
         {{- range $config.args }}
         - {{ . }}
         {{- end }}
+        {{- if and (eq $.Values.global.image.type "pack") $config.args }}
+        command:
+        - launcher
+        {{- end }}
         {{- if hasKey $config "web" }}
         env:
         - name: PORT

plugins/scheduler-k3s/triggers.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ryanuber/columnize"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/kubectl/pkg/util/term"
 	"k8s.io/kubernetes/pkg/client/conditions"
 )
 
@@ -281,7 +282,7 @@ func TriggerSchedulerCronWrite(scheduler string, appName string) error {
 	for _, cronTask := range cronTasks {
 		labelSelector := []string{
 			fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
-			fmt.Sprintf("dokku.com/cron-id=%s", cronTask.ID),
+			fmt.Sprintf("dokku.com/cron-hash=%s", cronIDLabelValue(cronTask.ID)),
 		}
 
 		if cronTask.Maintenance {
@@ -789,7 +790,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		// todo: implement pod annotations
 		suffix := ""
 		for _, cronJob := range cronJobs {
-			if cronJob.Labels["dokku.com/cron-id"] == cronTask.ID {
+			if cronJob.Annotations["dokku.com/cron-id"] == cronTask.ID {
 				var ok bool
 				suffix, ok = cronJob.Annotations["dokku.com/job-suffix"]
 				if !ok {
@@ -842,6 +843,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			Annotations: annotations,
 			Cron: ProcessCron{
 				ID:                cronTask.ID,
+				Hash:              cronIDLabelValue(cronTask.ID),
 				Schedule:          cronTask.Schedule,
 				Suffix:            suffix,
 				Suspend:           cronTask.Maintenance,
@@ -1367,26 +1369,25 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	processType := "run"
 	if os.Getenv("DOKKU_CRON_ID") != "" {
 		processType = "cron"
-		labels["dokku.com/cron-id"] = os.Getenv("DOKKU_CRON_ID")
+		cronHash := cronIDLabelValue(os.Getenv("DOKKU_CRON_ID"))
+		labels["dokku.com/cron-hash"] = cronHash
 		concurrencyPolicy := strings.ToUpper(os.Getenv("DOKKU_CONCURRENCY_POLICY"))
 		switch concurrencyPolicy {
 		case "forbid":
-			// check if there is a running pod with the same dokku.com/cron-id label
 			pods, err := clientset.ListPods(context.Background(), ListPodsInput{
 				Namespace:     namespace,
-				LabelSelector: fmt.Sprintf("dokku.com/cron-id=%s", os.Getenv("DOKKU_CRON_ID")),
+				LabelSelector: fmt.Sprintf("dokku.com/cron-hash=%s", cronHash),
 			})
 			if err != nil {
 				return fmt.Errorf("Error listing pods: %w", err)
 			}
 			if len(pods) > 0 {
-				return fmt.Errorf("There is a running pod with the same dokku.com/cron-id label")
+				return fmt.Errorf("There is a running pod with the same dokku.com/cron-hash label")
 			}
 		case "replace":
-			// delete any existing pod with the same dokku.com/cron-id label
 			err := clientset.DeletePod(context.Background(), DeletePodInput{
 				Namespace:     namespace,
-				LabelSelector: fmt.Sprintf("dokku.com/cron-id=%s", os.Getenv("DOKKU_CRON_ID")),
+				LabelSelector: fmt.Sprintf("dokku.com/cron-hash=%s", cronHash),
 			})
 			if err != nil {
 				return fmt.Errorf("Error deleting pod: %w", err)
@@ -1409,16 +1410,22 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 			Trigger: "procfile-get-command",
 			Args:    []string{appName, args[0], "5000"},
 		})
-		if err == nil && resp.Stdout != "" {
+		if err == nil && resp.StdoutContents() != "" {
 			common.LogInfo1Quiet(fmt.Sprintf("Found '%s' in Procfile, running that command", args[0]))
-			return err
+			words, err := shellquote.Split(resp.StdoutContents())
+			if err != nil {
+				return fmt.Errorf("Error parsing Procfile command: %w", err)
+			}
+			command = words
 		}
-		// todo: run command in procfile
 	}
 
 	entrypoint := ""
-	if imageSourceType == "herokuish" {
+	switch imageSourceType {
+	case "herokuish":
 		entrypoint = "/exec"
+	case "pack":
+		entrypoint = "launcher"
 	}
 
 	helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
@@ -1450,6 +1457,12 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 
 	attachToPod := os.Getenv("DOKKU_DETACH_CONTAINER") != "1"
 
+	forceTTY := common.ToBool(os.Getenv("DOKKU_FORCE_TTY"))
+	disableTTY := common.ToBool(os.Getenv("DOKKU_DISABLE_TTY"))
+	ttyProbe := term.TTY{Out: os.Stdout}
+	hasTTY := ttyProbe.MonitorSize(ttyProbe.GetSize()) != nil
+	streamLogsOnly := attachToPod && !forceTTY && (disableTTY || !hasTTY)
+
 	imagePullSecrets := getComputedImagePullSecrets(appName)
 	if imagePullSecrets == "" {
 		imagePullSecrets = GetImagePullSecretName(appName)
@@ -1482,7 +1495,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	workingDir := common.GetWorkingDir(appName, image)
 	job, err := templateKubernetesJob(Job{
 		AppName:               appName,
-		Command:               []string{commandShell},
+		Command:               command,
 		DeploymentID:          deploymentID,
 		Entrypoint:            entrypoint,
 		Env:                   extraEnv,
@@ -1559,9 +1572,45 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		Clientset:     clientset,
 		Namespace:     namespace,
 		LabelSelector: batchJobSelector,
-		Timeout:       10,
+		Timeout:       30,
 		Waiter:        isPodReady,
 	})
+	if streamLogsOnly && (err == nil || errors.Is(err, conditions.ErrPodCompleted)) {
+		if streamErr := clientset.StreamLogs(ctx, StreamLogsInput{
+			Follow:        true,
+			LabelSelector: []string{batchJobSelector},
+			Namespace:     namespace,
+			Quiet:         true,
+		}); streamErr != nil {
+			return fmt.Errorf("Error streaming logs: %w", streamErr)
+		}
+
+		selectedPod, waitErr := waitForPodBySelectorCompleted(ctx, WaitForPodBySelectorCompletedInput{
+			Clientset:     clientset,
+			Namespace:     namespace,
+			LabelSelector: batchJobSelector,
+			Timeout:       10,
+		})
+		if waitErr != nil {
+			return fmt.Errorf("Pod did not reach terminal state after logs streamed: %w", waitErr)
+		}
+		switch selectedPod.Status.Phase {
+		case v1.PodSucceeded:
+			return nil
+		case v1.PodFailed:
+			for _, status := range selectedPod.Status.ContainerStatuses {
+				if status.Name != fmt.Sprintf("%s-%s", appName, processType) {
+					continue
+				}
+				if status.State.Terminated == nil {
+					continue
+				}
+				return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code: %s", status.State.Terminated.Message)
+			}
+			return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code")
+		}
+		return fmt.Errorf("Unable to attach as the pod is in an unknown state: %s", selectedPod.Status.Phase)
+	}
 	if err != nil {
 		if errors.Is(err, conditions.ErrPodCompleted) {
 			pods, podErr := clientset.ListPods(ctx, ListPodsInput{
@@ -1699,9 +1748,9 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er
 			}
 		}
 
-		cronID, ok := cronJob.Labels["dokku.com/cron-id"]
+		cronID, ok := cronJob.Annotations["dokku.com/cron-id"]
 		if !ok {
-			common.LogWarn(fmt.Sprintf("Cron job %s does not have a cron ID label", cronJob.Name))
+			common.LogWarn(fmt.Sprintf("Cron job %s does not have a cron ID annotation", cronJob.Name))
 			continue
 		}
 

tests/apps/dockerfile-procfile/app-cron-procfile.json

@@ -0,0 +1,21 @@
+{
+  "cron": [
+    {
+      "command": "release",
+      "schedule": "5 5 5 5 5"
+    }
+  ],
+  "healthchecks": {
+    "web": [
+      {
+        "attempts": 2,
+        "content": "nodejs/express",
+        "name": "check-1",
+        "path": "/",
+        "timeout": 5,
+        "type": "startup",
+        "wait": 2
+      }
+    ]
+  }
+}

tests/apps/dockerfile-procfile/app-cron.json

@@ -0,0 +1,27 @@
+{
+  "cron": [
+    {
+      "command": "echo hello-from-cron",
+      "schedule": "5 5 5 5 5"
+    }
+  ],
+  "healthchecks": {
+    "web": [
+      {
+        "attempts": 2,
+        "content": "nodejs/express",
+        "name": "check-1",
+        "path": "/",
+        "timeout": 5,
+        "type": "startup",
+        "wait": 2
+      }
+    ]
+  },
+  "scripts": {
+    "dokku": {
+      "postdeploy": "touch /app/postdeploy.test",
+      "predeploy": "touch /app/predeploy.test"
+    }
+  }
+}

tests/apps/python/app-cnb-cron.json

@@ -1,7 +1,7 @@
 {
   "cron": [
     {
-      "command": "python3 task.py some cron task",
+      "command": "python3 task.py",
       "schedule": "5 5 5 5 5"
     }
   ],

tests/apps/python/app-cron-procfile.json

@@ -0,0 +1,21 @@
+{
+  "cron": [
+    {
+      "command": "task",
+      "schedule": "5 5 5 5 5"
+    }
+  ],
+  "healthchecks": {
+    "web": [
+      {
+        "attempts": 2,
+        "content": "python/http.server",
+        "name": "check-1",
+        "path": "/",
+        "timeout": 5,
+        "type": "startup",
+        "wait": 2
+      }
+    ]
+  }
+}

tests/unit/builder-pack.bats

@@ -216,7 +216,7 @@ teardown() {
   echo "output: $output"
   echo "status: $status"
   assert_success
-  assert_output "['task.py', 'some', 'cron', 'task']"
+  assert_output "['task.py']"
 }
 
 @test "(builder-pack) cron:run with Procfile reference" {

tests/unit/scheduler-k3s-cnb.bats

@@ -0,0 +1,275 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup_file() {
+  install_pack
+}
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s) cnb deployment sets command launcher for non-web process" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains 'from cnb stack'
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "launcher"
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "python3"
+}
+
+@test "(scheduler-k3s) cnb cronjob manifest sets command launcher" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains 'from cnb stack'
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "launcher"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py']"
+}
+
+@test "(scheduler-k3s) cnb dokku run uses launcher entrypoint" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains 'from cnb stack'
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP python3 task.py test"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP env"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY=fjdkslafjdk"
+}
+
+@test "(scheduler-k3s) cnb dokku run resolves Procfile key" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains 'from cnb stack'
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP task"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+}
+
+@test "(scheduler-k3s) cnb dokku run:detached returns pod name with launcher entrypoint" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains 'from cnb stack'
+
+  run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+  pod_name="$output"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$TEST_APP"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "pack"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "launcher"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "sleep"
+}
+
+cron_run_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  add_requirements_txt "$APP" "$APP_REPO_DIR"
+  mv -f "$APP_REPO_DIR/app-cnb-cron.json" "$APP_REPO_DIR/app.json"
+}
+
+cron_run_procfile_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  add_requirements_txt "$APP" "$APP_REPO_DIR"
+  mv -f "$APP_REPO_DIR/app-cnb-cron-procfile.json" "$APP_REPO_DIR/app.json"
+}

tests/unit/scheduler-k3s-dockerfile.bats

@@ -0,0 +1,240 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s) dockerfile deployment sets command for non-web process" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "node"
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[1]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "worker.js"
+}
+
+@test "(scheduler-k3s) dockerfile cronjob manifest sets command" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "echo"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "hello-from-cron"
+}
+
+@test "(scheduler-k3s) dockerfile dokku run executes command" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP echo hello-from-run"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "hello-from-run"
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP env"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY=fjdkslafjdk"
+}
+
+@test "(scheduler-k3s) dockerfile dokku run resolves Procfile key" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP release"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY: fjdkslafjdk"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY: fjdkslafjdk"
+}
+
+@test "(scheduler-k3s) dockerfile dokku run:detached returns pod name" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+  pod_name="$output"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$TEST_APP"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dockerfile"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+}
+
+cron_run_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  mv -f "$APP_REPO_DIR/app-cron.json" "$APP_REPO_DIR/app.json"
+}
+
+cron_run_procfile_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  mv -f "$APP_REPO_DIR/app-cron-procfile.json" "$APP_REPO_DIR/app.json"
+}

tests/unit/scheduler-k3s-herokuish.bats

@@ -0,0 +1,247 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s) herokuish deployment sets start command for non-web process" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "/start"
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[1]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "worker"
+}
+
+@test "(scheduler-k3s) herokuish cronjob manifest sets start command" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "python3"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'some', 'cron', 'task']"
+}
+
+@test "(scheduler-k3s) herokuish dokku run uses /exec entrypoint" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP python3 task.py test"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP env"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY=fjdkslafjdk"
+}
+
+@test "(scheduler-k3s) herokuish dokku run resolves Procfile key" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP task"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+}
+
+@test "(scheduler-k3s) herokuish dokku run:detached returns pod name" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+  pod_name="$output"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$TEST_APP"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "herokuish"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "/exec"
+}
+
+cron_run_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  add_requirements_txt "$APP" "$APP_REPO_DIR"
+  mv -f "$APP_REPO_DIR/app-cron.json" "$APP_REPO_DIR/app.json"
+}
+
+cron_run_procfile_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  add_requirements_txt "$APP" "$APP_REPO_DIR"
+  mv -f "$APP_REPO_DIR/app-cron-procfile.json" "$APP_REPO_DIR/app.json"
+}