diff --git a/plugins/scheduler-k3s/cron_job_template_test.go b/plugins/scheduler-k3s/cron_job_template_test.go new file mode 100644 index 000000000..4770f7cb8 --- /dev/null +++ b/plugins/scheduler-k3s/cron_job_template_test.go @@ -0,0 +1,149 @@ +package scheduler_k3s + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/engine" +) + +// TestCronIDLabelValue asserts the hashed cron ID fits inside Kubernetes' 63 +// byte label cap and is deterministic. Without this the label can exceed the +// cap and the Kubernetes API server rejects the manifest. +func TestCronIDLabelValue(t *testing.T) { + cases := []string{ + "short", + "app===echo hello-from-cron===5 5 5 5 5", + strings.Repeat("a-very-long-cron-id-that-would-far-exceed-the-label-cap-", 10), + } + for _, in := range cases { + out := cronIDLabelValue(in) + if len(out) > 63 { + t.Errorf("cronIDLabelValue(%q) = %q (len %d); must be <= 63", in, out, len(out)) + } + if out != cronIDLabelValue(in) { + t.Errorf("cronIDLabelValue(%q) is not deterministic", in) + } + } +} + +// TestCronJobTemplateQuotesAllDigitSuffix asserts that the rendered cron-job +// manifest produces string-typed annotation values even when the suffix and +// cron-id consist entirely of digits. Without `| quote` in the template, YAML +// would coerce these to numbers and the manifest would be rejected by the +// Kubernetes API server. +func TestCronJobTemplateQuotesAllDigitSuffix(t *testing.T) { + chartDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n") + if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil { + t.Fatalf("write Chart.yaml: %v", err) + } + + cronJobTpl, err := templates.ReadFile("templates/chart/cron-job.yaml") + if err != nil { + t.Fatalf("read cron-job template: %v", err) + } + if err := os.WriteFile(filepath.Join(chartDir, "templates", "cron-job.yaml"), cronJobTpl, 0o644); err != nil { + t.Fatalf("write cron-job template: %v", err) + } + + helpersTpl, err := templates.ReadFile("templates/chart/_helpers.tpl") + if err != nil { + t.Fatalf("read _helpers: %v", err) + } + if err := os.WriteFile(filepath.Join(chartDir, "templates", "_helpers.tpl"), helpersTpl, 0o644); err != nil { + t.Fatalf("write _helpers: %v", err) + } + + loaded, err := loader.Load(chartDir) + if err != nil { + t.Fatalf("load chart: %v", err) + } + + values := map[string]interface{}{ + "global": map[string]interface{}{ + "app_name": "myapp", + "deployment_id": "1", + "namespace": "myapp", + "image": map[string]interface{}{ + "name": "myapp:latest", + "type": "dockerfile", + }, + }, + "processes": map[string]interface{}{ + "cron-id-123": map[string]interface{}{ + "args": []interface{}{"echo", "hello"}, + "cron": map[string]interface{}{ + "id": "1234567890", + "hash": "abc123def", + "schedule": "5 5 5 5 5", + "suffix": "1234567890", + "suspend": false, + "concurrency_policy": "Allow", + }, + }, + }, + } + + renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil) + if err != nil { + t.Fatalf("ToRenderValues: %v", err) + } + + rendered, err := engine.Render(loaded, renderValues) + if err != nil { + t.Fatalf("render: %v", err) + } + + var manifest string + for name, content := range rendered { + if filepath.Base(name) == "cron-job.yaml" { + manifest = content + break + } + } + if manifest == "" { + t.Fatalf("cron-job.yaml not rendered; got: %v", rendered) + } + + decoder := yaml.NewDecoder(strings.NewReader(manifest)) + for { + var doc map[string]interface{} + if err := decoder.Decode(&doc); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("yaml decode failed (would also fail in Kubernetes API): %v\nrendered:\n%s", err, manifest) + } + if doc == nil { + continue + } + metadata, ok := doc["metadata"].(map[string]interface{}) + if !ok { + continue + } + annotations, _ := metadata["annotations"].(map[string]interface{}) + for key, value := range annotations { + if _, isString := value.(string); !isString { + t.Errorf("annotation %q has non-string value %v (type %T); helm template must apply | quote", key, value, value) + } + } + labels, _ := metadata["labels"].(map[string]interface{}) + for key, value := range labels { + if _, isString := value.(string); !isString { + t.Errorf("label %q has non-string value %v (type %T); helm template must apply | quote", key, value, value) + } + } + } +} diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index faaddffca..a183efe7b 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -3,6 +3,7 @@ package scheduler_k3s import ( "bytes" "context" + "crypto/sha256" "encoding/base64" "errors" "fmt" @@ -23,6 +24,7 @@ import ( nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts" resty "github.com/go-resty/resty/v2" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/multiformats/go-base36" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" "helm.sh/helm/v3/pkg/strvals" @@ -134,6 +136,23 @@ type WaitForPodBySelectorRunningInput struct { Waiter func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc } +type WaitForPodBySelectorCompletedInput struct { + // Clientset is the kubernetes clientset + Clientset KubernetesClient + + // Namespace is the namespace to search in + Namespace string + + // LabelSelector is the label selector to search for + LabelSelector string + + // PodName is the pod name to search for + PodName string + + // Timeout is the timeout in seconds to wait for the pod to reach a terminal phase + Timeout float64 +} + type WaitForPodToExistInput struct { Clientset KubernetesClient Namespace string @@ -1981,6 +2000,52 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp } } +// waitForPodBySelectorCompleted polls the first pod matching the input +// selector until it reaches a terminal phase (PodSucceeded / PodFailed) or +// the timeout elapses. Used after a Follow=true StreamLogs returns, to +// handle the kubelet -> apiserver phase propagation delay for short-lived +// run pods. +func waitForPodBySelectorCompleted(ctx context.Context, input WaitForPodBySelectorCompletedInput) (v1.Pod, error) { + pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{ + Clientset: input.Clientset, + LabelSelector: input.LabelSelector, + Namespace: input.Namespace, + PodName: input.PodName, + RetryCount: 3, + }) + if err != nil { + return v1.Pod{}, fmt.Errorf("Error waiting for pod to exist: %w", err) + } + + if len(pods) == 0 { + return v1.Pod{}, fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.LabelSelector) + } + + podName := pods[0].Name + if input.PodName != "" { + podName = input.PodName + } + + timeout := time.Duration(input.Timeout * float64(time.Second)) + var pod v1.Pod + err = wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + p, err := input.Clientset.GetPod(ctx, GetPodInput{ + Name: podName, + Namespace: input.Namespace, + }) + if err != nil { + return false, err + } + pod = p + switch p.Status.Phase { + case v1.PodSucceeded, v1.PodFailed: + return true, nil + } + return false, nil + }) + return pod, err +} + // kubernetesNodeToNode converts a kubernetes node to a Node func kubernetesNodeToNode(node v1.Node) Node { roles := []string{} @@ -2018,6 +2083,14 @@ func kubernetesNodeToNode(node v1.Node) Node { } } +// cronIDLabelValue returns a Kubernetes-label-safe hash of a cron ID. +// The raw cron ID can exceed the 63-byte label cap, so we keep it as an +// annotation and use this short hash for selectors. +func cronIDLabelValue(cronID string) string { + sum := sha256.Sum256([]byte(cronID)) + return base36.EncodeToStringLc(sum[:16]) +} + // parseMemoryQuantity parses a string into a valid memory quantity func parseMemoryQuantity(input string) (string, error) { if _, err := strconv.ParseInt(input, 10, 64); err == nil { diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index 1d375ec86..ea594fefc 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -22,6 +22,7 @@ require ( github.com/gosimple/slug v1.15.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kedacore/keda/v2 v2.18.3 + github.com/multiformats/go-base36 v0.2.0 github.com/ryanuber/columnize v2.1.2+incompatible github.com/spf13/pflag v1.0.10 github.com/traefik/traefik/v2 v2.11.46 @@ -115,7 +116,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect - github.com/multiformats/go-base36 v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 9d2b2e63a..554bde5f4 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -305,6 +305,7 @@ const ( type ProcessCron struct { ID string `yaml:"id"` + Hash string `yaml:"hash"` Schedule string `yaml:"schedule"` Suffix string `yaml:"suffix"` Suspend bool `yaml:"suspend"` diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index e8cb4675a..3d4ee9b79 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -9,18 +9,18 @@ kind: CronJob metadata: annotations: app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} - dokku.com/builder-type: {{ $.Values.global.image.type }} - dokku.com/cron-id: {{ $config.cron.id }} - dokku.com/job-suffix: {{ $config.cron.suffix }} + dokku.com/builder-type: {{ $.Values.global.image.type | quote }} + dokku.com/cron-id: {{ $config.cron.id | quote }} + dokku.com/job-suffix: {{ $config.cron.suffix | quote }} dokku.com/managed: "true" - kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + kubectl.kubernetes.io/default-container: {{ printf "%s-cron" $.Values.global.app_name | quote }} {{ include "print.annotations" (dict "config" $.Values.global "key" "cronjob") | indent 4 }} {{ include "print.annotations" (dict "config" $config "key" "cronjob") | indent 4 }} labels: - app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} + app.kubernetes.io/instance: {{ printf "%s-cron-%s" $.Values.global.app_name $config.cron.suffix | quote }} app.kubernetes.io/name: cron - app.kubernetes.io/part-of: {{ $.Values.global.app_name }} - dokku.com/cron-id: {{ $config.cron.id }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }} + dokku.com/cron-hash: {{ $config.cron.hash | quote }} {{ include "print.labels" (dict "config" $.Values.global "key" "cronjob") | indent 4 }} {{ include "print.labels" (dict "config" $config "key" "cronjob") | indent 4 }} name: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} @@ -32,18 +32,18 @@ spec: metadata: annotations: app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} - dokku.com/builder-type: {{ $.Values.global.image.type }} - dokku.com/cron-id: {{ $config.cron.id }} - dokku.com/job-suffix: {{ $config.cron.suffix }} + dokku.com/builder-type: {{ $.Values.global.image.type | quote }} + dokku.com/cron-id: {{ $config.cron.id | quote }} + dokku.com/job-suffix: {{ $config.cron.suffix | quote }} dokku.com/managed: "true" - kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + kubectl.kubernetes.io/default-container: {{ printf "%s-cron" $.Values.global.app_name | quote }} {{ include "print.annotations" (dict "config" $.Values.global "key" "job") | indent 8 }} {{ include "print.annotations" (dict "config" $config "key" "job") | indent 8 }} labels: - app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} + app.kubernetes.io/instance: {{ printf "%s-cron-%s" $.Values.global.app_name $config.cron.suffix | quote }} app.kubernetes.io/name: cron - app.kubernetes.io/part-of: {{ $.Values.global.app_name }} - dokku.com/cron-id: {{ $config.cron.id }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }} + dokku.com/cron-hash: {{ $config.cron.hash | quote }} {{ include "print.labels" (dict "config" $.Values.global "key" "job") | indent 8 }} {{ include "print.labels" (dict "config" $config "key" "job") | indent 8 }} spec: @@ -55,18 +55,18 @@ spec: metadata: annotations: app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} - dokku.com/builder-type: {{ $.Values.global.image.type }} - dokku.com/cron-id: {{ $config.cron.id }} - dokku.com/job-suffix: {{ $config.cron.suffix }} + dokku.com/builder-type: {{ $.Values.global.image.type | quote }} + dokku.com/cron-id: {{ $config.cron.id | quote }} + dokku.com/job-suffix: {{ $config.cron.suffix | quote }} dokku.com/managed: "true" - kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + kubectl.kubernetes.io/default-container: {{ printf "%s-cron" $.Values.global.app_name | quote }} {{ include "print.annotations" (dict "config" $.Values.global "key" "pod") | indent 12 }} {{ include "print.annotations" (dict "config" $config "key" "pod") | indent 12 }} labels: - app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} + app.kubernetes.io/instance: {{ printf "%s-cron-%s" $.Values.global.app_name $config.cron.suffix | quote }} app.kubernetes.io/name: cron - app.kubernetes.io/part-of: {{ $.Values.global.app_name }} - dokku.com/cron-id: {{ $config.cron.id }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }} + dokku.com/cron-hash: {{ $config.cron.hash | quote }} {{ include "print.labels" (dict "config" $.Values.global "key" "pod") | indent 12 }} {{ include "print.labels" (dict "config" $config "key" "pod") | indent 12 }} spec: @@ -75,6 +75,10 @@ spec: {{- range $config.args }} - {{ . }} {{- end }} + {{- if eq $.Values.global.image.type "pack" }} + command: + - launcher + {{- end }} envFrom: - secretRef: name: config-{{ $.Values.global.app_name }} diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index bee80b421..34312777e 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -16,15 +16,15 @@ kind: Deployment metadata: annotations: app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} - dokku.com/builder-type: {{ $.Values.global.image.type }} + dokku.com/builder-type: {{ $.Values.global.image.type | quote }} dokku.com/managed: "true" - kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }} + kubectl.kubernetes.io/default-container: {{ printf "%s-%s" $.Values.global.app_name $processName | quote }} {{ include "print.annotations" (dict "config" $.Values.global "key" "deployment") | indent 4 }} {{ include "print.annotations" (dict "config" $config "key" "deployment") | indent 4 }} labels: - app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} - app.kubernetes.io/name: {{ $processName }} - app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + app.kubernetes.io/instance: {{ printf "%s-%s" $.Values.global.app_name $processName | quote }} + app.kubernetes.io/name: {{ $processName | quote }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name | quote }} {{ include "print.labels" (dict "config" $.Values.global "key" "deployment") | indent 4 }} {{ include "print.labels" (dict "config" $config "key" "deployment") | indent 4 }} name: {{ $.Values.global.app_name }}-{{ $processName }} @@ -63,6 +63,10 @@ spec: {{- range $config.args }} - {{ . }} {{- end }} + {{- if and (eq $.Values.global.image.type "pack") $config.args }} + command: + - launcher + {{- end }} {{- if hasKey $config "web" }} env: - name: PORT diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index ec455a304..d1e01f360 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -29,6 +29,7 @@ import ( "github.com/ryanuber/columnize" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + "k8s.io/kubectl/pkg/util/term" "k8s.io/kubernetes/pkg/client/conditions" ) @@ -281,7 +282,7 @@ func TriggerSchedulerCronWrite(scheduler string, appName string) error { for _, cronTask := range cronTasks { labelSelector := []string{ fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), - fmt.Sprintf("dokku.com/cron-id=%s", cronTask.ID), + fmt.Sprintf("dokku.com/cron-hash=%s", cronIDLabelValue(cronTask.ID)), } if cronTask.Maintenance { @@ -789,7 +790,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e // todo: implement pod annotations suffix := "" for _, cronJob := range cronJobs { - if cronJob.Labels["dokku.com/cron-id"] == cronTask.ID { + if cronJob.Annotations["dokku.com/cron-id"] == cronTask.ID { var ok bool suffix, ok = cronJob.Annotations["dokku.com/job-suffix"] if !ok { @@ -842,6 +843,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Annotations: annotations, Cron: ProcessCron{ ID: cronTask.ID, + Hash: cronIDLabelValue(cronTask.ID), Schedule: cronTask.Schedule, Suffix: suffix, Suspend: cronTask.Maintenance, @@ -1367,26 +1369,25 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] processType := "run" if os.Getenv("DOKKU_CRON_ID") != "" { processType = "cron" - labels["dokku.com/cron-id"] = os.Getenv("DOKKU_CRON_ID") + cronHash := cronIDLabelValue(os.Getenv("DOKKU_CRON_ID")) + labels["dokku.com/cron-hash"] = cronHash concurrencyPolicy := strings.ToUpper(os.Getenv("DOKKU_CONCURRENCY_POLICY")) switch concurrencyPolicy { case "forbid": - // check if there is a running pod with the same dokku.com/cron-id label pods, err := clientset.ListPods(context.Background(), ListPodsInput{ Namespace: namespace, - LabelSelector: fmt.Sprintf("dokku.com/cron-id=%s", os.Getenv("DOKKU_CRON_ID")), + LabelSelector: fmt.Sprintf("dokku.com/cron-hash=%s", cronHash), }) if err != nil { return fmt.Errorf("Error listing pods: %w", err) } if len(pods) > 0 { - return fmt.Errorf("There is a running pod with the same dokku.com/cron-id label") + return fmt.Errorf("There is a running pod with the same dokku.com/cron-hash label") } case "replace": - // delete any existing pod with the same dokku.com/cron-id label err := clientset.DeletePod(context.Background(), DeletePodInput{ Namespace: namespace, - LabelSelector: fmt.Sprintf("dokku.com/cron-id=%s", os.Getenv("DOKKU_CRON_ID")), + LabelSelector: fmt.Sprintf("dokku.com/cron-hash=%s", cronHash), }) if err != nil { return fmt.Errorf("Error deleting pod: %w", err) @@ -1409,16 +1410,22 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] Trigger: "procfile-get-command", Args: []string{appName, args[0], "5000"}, }) - if err == nil && resp.Stdout != "" { + if err == nil && resp.StdoutContents() != "" { common.LogInfo1Quiet(fmt.Sprintf("Found '%s' in Procfile, running that command", args[0])) - return err + words, err := shellquote.Split(resp.StdoutContents()) + if err != nil { + return fmt.Errorf("Error parsing Procfile command: %w", err) + } + command = words } - // todo: run command in procfile } entrypoint := "" - if imageSourceType == "herokuish" { + switch imageSourceType { + case "herokuish": entrypoint = "/exec" + case "pack": + entrypoint = "launcher" } helmAgent, err := NewHelmAgent(namespace, DevNullPrinter) @@ -1450,6 +1457,12 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] attachToPod := os.Getenv("DOKKU_DETACH_CONTAINER") != "1" + forceTTY := common.ToBool(os.Getenv("DOKKU_FORCE_TTY")) + disableTTY := common.ToBool(os.Getenv("DOKKU_DISABLE_TTY")) + ttyProbe := term.TTY{Out: os.Stdout} + hasTTY := ttyProbe.MonitorSize(ttyProbe.GetSize()) != nil + streamLogsOnly := attachToPod && !forceTTY && (disableTTY || !hasTTY) + imagePullSecrets := getComputedImagePullSecrets(appName) if imagePullSecrets == "" { imagePullSecrets = GetImagePullSecretName(appName) @@ -1482,7 +1495,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] workingDir := common.GetWorkingDir(appName, image) job, err := templateKubernetesJob(Job{ AppName: appName, - Command: []string{commandShell}, + Command: command, DeploymentID: deploymentID, Entrypoint: entrypoint, Env: extraEnv, @@ -1559,9 +1572,45 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] Clientset: clientset, Namespace: namespace, LabelSelector: batchJobSelector, - Timeout: 10, + Timeout: 30, Waiter: isPodReady, }) + if streamLogsOnly && (err == nil || errors.Is(err, conditions.ErrPodCompleted)) { + if streamErr := clientset.StreamLogs(ctx, StreamLogsInput{ + Follow: true, + LabelSelector: []string{batchJobSelector}, + Namespace: namespace, + Quiet: true, + }); streamErr != nil { + return fmt.Errorf("Error streaming logs: %w", streamErr) + } + + selectedPod, waitErr := waitForPodBySelectorCompleted(ctx, WaitForPodBySelectorCompletedInput{ + Clientset: clientset, + Namespace: namespace, + LabelSelector: batchJobSelector, + Timeout: 10, + }) + if waitErr != nil { + return fmt.Errorf("Pod did not reach terminal state after logs streamed: %w", waitErr) + } + switch selectedPod.Status.Phase { + case v1.PodSucceeded: + return nil + case v1.PodFailed: + for _, status := range selectedPod.Status.ContainerStatuses { + if status.Name != fmt.Sprintf("%s-%s", appName, processType) { + continue + } + if status.State.Terminated == nil { + continue + } + return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code: %s", status.State.Terminated.Message) + } + return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code") + } + return fmt.Errorf("Unable to attach as the pod is in an unknown state: %s", selectedPod.Status.Phase) + } if err != nil { if errors.Is(err, conditions.ErrPodCompleted) { pods, podErr := clientset.ListPods(ctx, ListPodsInput{ @@ -1699,9 +1748,9 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er } } - cronID, ok := cronJob.Labels["dokku.com/cron-id"] + cronID, ok := cronJob.Annotations["dokku.com/cron-id"] if !ok { - common.LogWarn(fmt.Sprintf("Cron job %s does not have a cron ID label", cronJob.Name)) + common.LogWarn(fmt.Sprintf("Cron job %s does not have a cron ID annotation", cronJob.Name)) continue } diff --git a/tests/apps/dockerfile-procfile/app-cron-procfile.json b/tests/apps/dockerfile-procfile/app-cron-procfile.json new file mode 100644 index 000000000..22ed426fd --- /dev/null +++ b/tests/apps/dockerfile-procfile/app-cron-procfile.json @@ -0,0 +1,21 @@ +{ + "cron": [ + { + "command": "release", + "schedule": "5 5 5 5 5" + } + ], + "healthchecks": { + "web": [ + { + "attempts": 2, + "content": "nodejs/express", + "name": "check-1", + "path": "/", + "timeout": 5, + "type": "startup", + "wait": 2 + } + ] + } +} diff --git a/tests/apps/dockerfile-procfile/app-cron.json b/tests/apps/dockerfile-procfile/app-cron.json new file mode 100644 index 000000000..2b7361dbb --- /dev/null +++ b/tests/apps/dockerfile-procfile/app-cron.json @@ -0,0 +1,27 @@ +{ + "cron": [ + { + "command": "echo hello-from-cron", + "schedule": "5 5 5 5 5" + } + ], + "healthchecks": { + "web": [ + { + "attempts": 2, + "content": "nodejs/express", + "name": "check-1", + "path": "/", + "timeout": 5, + "type": "startup", + "wait": 2 + } + ] + }, + "scripts": { + "dokku": { + "postdeploy": "touch /app/postdeploy.test", + "predeploy": "touch /app/predeploy.test" + } + } +} diff --git a/tests/apps/python/app-cnb-cron.json b/tests/apps/python/app-cnb-cron.json index 54d9aa7a1..67943099b 100644 --- a/tests/apps/python/app-cnb-cron.json +++ b/tests/apps/python/app-cnb-cron.json @@ -1,7 +1,7 @@ { "cron": [ { - "command": "python3 task.py some cron task", + "command": "python3 task.py", "schedule": "5 5 5 5 5" } ], diff --git a/tests/apps/python/app-cron-procfile.json b/tests/apps/python/app-cron-procfile.json new file mode 100644 index 000000000..892bc0bf1 --- /dev/null +++ b/tests/apps/python/app-cron-procfile.json @@ -0,0 +1,21 @@ +{ + "cron": [ + { + "command": "task", + "schedule": "5 5 5 5 5" + } + ], + "healthchecks": { + "web": [ + { + "attempts": 2, + "content": "python/http.server", + "name": "check-1", + "path": "/", + "timeout": 5, + "type": "startup", + "wait": 2 + } + ] + } +} diff --git a/tests/unit/builder-pack.bats b/tests/unit/builder-pack.bats index d5dbc7887..a580a2575 100644 --- a/tests/unit/builder-pack.bats +++ b/tests/unit/builder-pack.bats @@ -216,7 +216,7 @@ teardown() { echo "output: $output" echo "status: $status" assert_success - assert_output "['task.py', 'some', 'cron', 'task']" + assert_output "['task.py']" } @test "(builder-pack) cron:run with Procfile reference" { diff --git a/tests/unit/scheduler-k3s-cnb.bats b/tests/unit/scheduler-k3s-cnb.bats new file mode 100644 index 000000000..4d54065f6 --- /dev/null +++ b/tests/unit/scheduler-k3s-cnb.bats @@ -0,0 +1,275 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup_file() { + install_pack +} + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s) cnb deployment sets command launcher for non-web process" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "launcher" + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "python3" +} + +@test "(scheduler-k3s) cnb cronjob manifest sets command launcher" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "launcher" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py']" +} + +@test "(scheduler-k3s) cnb dokku run uses launcher entrypoint" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "dokku --quiet run $TEST_APP python3 task.py test" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py', 'test']" + + run /bin/bash -c "dokku --quiet run $TEST_APP env" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "SECRET_KEY=fjdkslafjdk" +} + +@test "(scheduler-k3s) cnb dokku run resolves Procfile key" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "dokku --quiet run $TEST_APP task" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py', 'test']" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py', 'test']" +} + +@test "(scheduler-k3s) cnb dokku run:detached returns pod name with launcher entrypoint" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + pod_name="$output" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$TEST_APP" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "pack" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "launcher" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].args[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "sleep" +} + +cron_run_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + add_requirements_txt "$APP" "$APP_REPO_DIR" + mv -f "$APP_REPO_DIR/app-cnb-cron.json" "$APP_REPO_DIR/app.json" +} + +cron_run_procfile_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + add_requirements_txt "$APP" "$APP_REPO_DIR" + mv -f "$APP_REPO_DIR/app-cnb-cron-procfile.json" "$APP_REPO_DIR/app.json" +} diff --git a/tests/unit/scheduler-k3s-dockerfile.bats b/tests/unit/scheduler-k3s-dockerfile.bats new file mode 100644 index 000000000..6be1e49b0 --- /dev/null +++ b/tests/unit/scheduler-k3s-dockerfile.bats @@ -0,0 +1,240 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s) dockerfile deployment sets command for non-web process" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "node" + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[1]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "worker.js" +} + +@test "(scheduler-k3s) dockerfile cronjob manifest sets command" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" + + run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].args[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "echo" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "hello-from-cron" +} + +@test "(scheduler-k3s) dockerfile dokku run executes command" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku --quiet run $TEST_APP echo hello-from-run" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "hello-from-run" + + run /bin/bash -c "dokku --quiet run $TEST_APP env" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "SECRET_KEY=fjdkslafjdk" +} + +@test "(scheduler-k3s) dockerfile dokku run resolves Procfile key" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku --quiet run $TEST_APP release" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "SECRET_KEY: fjdkslafjdk" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "SECRET_KEY: fjdkslafjdk" +} + +@test "(scheduler-k3s) dockerfile dokku run:detached returns pod name" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app dockerfile dokku@$DOKKU_DOMAIN:$TEST_APP + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + pod_name="$output" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$TEST_APP" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "dockerfile" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" +} + +cron_run_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + mv -f "$APP_REPO_DIR/app-cron.json" "$APP_REPO_DIR/app.json" +} + +cron_run_procfile_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + mv -f "$APP_REPO_DIR/app-cron-procfile.json" "$APP_REPO_DIR/app.json" +} diff --git a/tests/unit/scheduler-k3s-herokuish.bats b/tests/unit/scheduler-k3s-herokuish.bats new file mode 100644 index 000000000..3f3786b41 --- /dev/null +++ b/tests/unit/scheduler-k3s-herokuish.bats @@ -0,0 +1,247 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s) herokuish deployment sets start command for non-web process" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "/start" + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[1]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "worker" +} + +@test "(scheduler-k3s) herokuish cronjob manifest sets start command" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" + + run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].args[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "python3" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py', 'some', 'cron', 'task']" +} + +@test "(scheduler-k3s) herokuish dokku run uses /exec entrypoint" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku --quiet run $TEST_APP python3 task.py test" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py', 'test']" + + run /bin/bash -c "dokku --quiet run $TEST_APP env" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "SECRET_KEY=fjdkslafjdk" +} + +@test "(scheduler-k3s) herokuish dokku run resolves Procfile key" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku --quiet run $TEST_APP task" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py', 'test']" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "['task.py', 'test']" +} + +@test "(scheduler-k3s) herokuish dokku run:detached returns pod name" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + pod_name="$output" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$TEST_APP" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "herokuish" + + run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "/exec" +} + +cron_run_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + add_requirements_txt "$APP" "$APP_REPO_DIR" + mv -f "$APP_REPO_DIR/app-cron.json" "$APP_REPO_DIR/app.json" +} + +cron_run_procfile_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + add_requirements_txt "$APP" "$APP_REPO_DIR" + mv -f "$APP_REPO_DIR/app-cron-procfile.json" "$APP_REPO_DIR/app.json" +}