From 3a845f4d145f577d3b799e2fe4f36b241c202069 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 30 Apr 2026 21:29:11 -0400 Subject: [PATCH] fix: route CNB images through launcher on scheduler-k3s Mirror the docker-local fix in #8525 for the k3s scheduler. CNB images default to a `/cnb/process/web` entrypoint that ignores incoming args, so non-web deployments, scheduled cron jobs, and ad-hoc `dokku run` / `cron:run` commands all need an explicit `launcher` entrypoint. The deployment and cron-job helm templates now set `command: [launcher]` when `image.type` is `pack`, and `TriggerSchedulerRun` sets the entrypoint to `launcher` for pack images while finishing the previously stubbed Procfile lookup branch so the resolved command is actually scheduled. --- .../templates/chart/cron-job.yaml | 4 + .../templates/chart/deployment.yaml | 4 + plugins/scheduler-k3s/triggers.go | 16 +- tests/unit/scheduler-k3s-cnb.bats | 226 ++++++++++++++++++ 4 files changed, 245 insertions(+), 5 deletions(-) create mode 100644 tests/unit/scheduler-k3s-cnb.bats diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index e8cb4675a..c3b9e30b4 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -75,6 +75,10 @@ spec: {{- range $config.args }} - {{ . }} {{- end }} + {{- if eq $.Values.global.image.type "pack" }} + command: + - launcher + {{- end }} envFrom: - secretRef: name: config-{{ $.Values.global.app_name }} diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index bee80b421..84c97dfcf 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -63,6 +63,10 @@ spec: {{- range $config.args }} - {{ . }} {{- end }} + {{- if and (eq $.Values.global.image.type "pack") $config.args }} + command: + - launcher + {{- end }} {{- if hasKey $config "web" }} env: - name: PORT diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index ec455a304..9b36a1869 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -1409,16 +1409,22 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] Trigger: "procfile-get-command", Args: []string{appName, args[0], "5000"}, }) - if err == nil && resp.Stdout != "" { + if err == nil && resp.StdoutContents() != "" { common.LogInfo1Quiet(fmt.Sprintf("Found '%s' in Procfile, running that command", args[0])) - return err + words, err := shellquote.Split(resp.StdoutContents()) + if err != nil { + return fmt.Errorf("Error parsing Procfile command: %w", err) + } + command = words } - // todo: run command in procfile } entrypoint := "" - if imageSourceType == "herokuish" { + switch imageSourceType { + case "herokuish": entrypoint = "/exec" + case "pack": + entrypoint = "launcher" } helmAgent, err := NewHelmAgent(namespace, DevNullPrinter) @@ -1482,7 +1488,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] workingDir := common.GetWorkingDir(appName, image) job, err := templateKubernetesJob(Job{ AppName: appName, - Command: []string{commandShell}, + Command: command, DeploymentID: deploymentID, Entrypoint: entrypoint, Env: extraEnv, diff --git a/tests/unit/scheduler-k3s-cnb.bats b/tests/unit/scheduler-k3s-cnb.bats new file mode 100644 index 000000000..588806f72 --- /dev/null +++ b/tests/unit/scheduler-k3s-cnb.bats @@ -0,0 +1,226 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup_file() { + install_pack +} + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s) cnb deployment sets command launcher for non-web process" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "launcher" + + run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "python3" + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o=jsonpath='{.spec.template.spec.containers[0].command}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" +} + +@test "(scheduler-k3s) cnb cronjob manifest sets command launcher" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command[0]}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "launcher" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "['task.py', 'some', 'cron', 'task']" +} + +@test "(scheduler-k3s) cnb dokku run uses launcher entrypoint" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "dokku --quiet run $TEST_APP python3 task.py test" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "['task.py', 'test']" + + run /bin/bash -c "dokku --quiet run $TEST_APP env" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "SECRET_KEY=fjdkslafjdk" +} + +@test "(scheduler-k3s) cnb dokku run resolves Procfile key" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'from cnb stack' + + run /bin/bash -c "dokku --quiet run $TEST_APP task" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "['task.py', 'test']" + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "['task.py', 'test']" +} + +cron_run_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + add_requirements_txt "$APP" "$APP_REPO_DIR" + mv -f "$APP_REPO_DIR/app-cnb-cron.json" "$APP_REPO_DIR/app.json" +} + +cron_run_procfile_wrapper() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + APP_REPO_DIR="$(realpath "$APP_REPO_DIR")" + + add_requirements_txt "$APP" "$APP_REPO_DIR" + mv -f "$APP_REPO_DIR/app-cnb-cron-procfile.json" "$APP_REPO_DIR/app.json" +}