fix: route CNB images through launcher on scheduler-k3s
Mirror the docker-local fix in #8525 for the k3s scheduler. CNB images default to a `/cnb/process/web` entrypoint that ignores incoming args, so non-web deployments, scheduled cron jobs, and ad-hoc `dokku run` / `cron:run` commands all need an explicit `launcher` entrypoint. The deployment and cron-job helm templates now set `command: [launcher]` when `image.type` is `pack`, and `TriggerSchedulerRun` sets the entrypoint to `launcher` for pack images while finishing the previously stubbed Procfile lookup branch so the resolved command is actually scheduled.
This commit is contained in:
@@ -75,6 +75,10 @@ spec:
|
|||||||
{{- range $config.args }}
|
{{- range $config.args }}
|
||||||
- {{ . }}
|
- {{ . }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- if eq $.Values.global.image.type "pack" }}
|
||||||
|
command:
|
||||||
|
- launcher
|
||||||
|
{{- end }}
|
||||||
envFrom:
|
envFrom:
|
||||||
- secretRef:
|
- secretRef:
|
||||||
name: config-{{ $.Values.global.app_name }}
|
name: config-{{ $.Values.global.app_name }}
|
||||||
|
|||||||
@@ -63,6 +63,10 @@ spec:
|
|||||||
{{- range $config.args }}
|
{{- range $config.args }}
|
||||||
- {{ . }}
|
- {{ . }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- if and (eq $.Values.global.image.type "pack") $config.args }}
|
||||||
|
command:
|
||||||
|
- launcher
|
||||||
|
{{- end }}
|
||||||
{{- if hasKey $config "web" }}
|
{{- if hasKey $config "web" }}
|
||||||
env:
|
env:
|
||||||
- name: PORT
|
- name: PORT
|
||||||
|
|||||||
@@ -1409,16 +1409,22 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
|
|||||||
Trigger: "procfile-get-command",
|
Trigger: "procfile-get-command",
|
||||||
Args: []string{appName, args[0], "5000"},
|
Args: []string{appName, args[0], "5000"},
|
||||||
})
|
})
|
||||||
if err == nil && resp.Stdout != "" {
|
if err == nil && resp.StdoutContents() != "" {
|
||||||
common.LogInfo1Quiet(fmt.Sprintf("Found '%s' in Procfile, running that command", args[0]))
|
common.LogInfo1Quiet(fmt.Sprintf("Found '%s' in Procfile, running that command", args[0]))
|
||||||
return err
|
words, err := shellquote.Split(resp.StdoutContents())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error parsing Procfile command: %w", err)
|
||||||
|
}
|
||||||
|
command = words
|
||||||
}
|
}
|
||||||
// todo: run command in procfile
|
|
||||||
}
|
}
|
||||||
|
|
||||||
entrypoint := ""
|
entrypoint := ""
|
||||||
if imageSourceType == "herokuish" {
|
switch imageSourceType {
|
||||||
|
case "herokuish":
|
||||||
entrypoint = "/exec"
|
entrypoint = "/exec"
|
||||||
|
case "pack":
|
||||||
|
entrypoint = "launcher"
|
||||||
}
|
}
|
||||||
|
|
||||||
helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
|
helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
|
||||||
@@ -1482,7 +1488,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
|
|||||||
workingDir := common.GetWorkingDir(appName, image)
|
workingDir := common.GetWorkingDir(appName, image)
|
||||||
job, err := templateKubernetesJob(Job{
|
job, err := templateKubernetesJob(Job{
|
||||||
AppName: appName,
|
AppName: appName,
|
||||||
Command: []string{commandShell},
|
Command: command,
|
||||||
DeploymentID: deploymentID,
|
DeploymentID: deploymentID,
|
||||||
Entrypoint: entrypoint,
|
Entrypoint: entrypoint,
|
||||||
Env: extraEnv,
|
Env: extraEnv,
|
||||||
|
|||||||
226
tests/unit/scheduler-k3s-cnb.bats
Normal file
226
tests/unit/scheduler-k3s-cnb.bats
Normal file
@@ -0,0 +1,226 @@
|
|||||||
|
#!/usr/bin/env bats
|
||||||
|
|
||||||
|
load test_helper
|
||||||
|
|
||||||
|
TEST_APP="rdmtestapp"
|
||||||
|
|
||||||
|
setup_file() {
|
||||||
|
install_pack
|
||||||
|
}
|
||||||
|
|
||||||
|
setup() {
|
||||||
|
uninstall_k3s || true
|
||||||
|
global_setup
|
||||||
|
dokku nginx:stop
|
||||||
|
export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
|
||||||
|
}
|
||||||
|
|
||||||
|
teardown() {
|
||||||
|
global_teardown
|
||||||
|
dokku nginx:start
|
||||||
|
uninstall_k3s || true
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "(scheduler-k3s) cnb deployment sets command launcher for non-web process" {
|
||||||
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||||
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
INGRESS_CLASS=nginx install_k3s
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku apps:create $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains 'from cnb stack'
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command[0]}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "launcher"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "python3"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o=jsonpath='{.spec.template.spec.containers[0].command}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output ""
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "(scheduler-k3s) cnb cronjob manifest sets command launcher" {
|
||||||
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||||
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
INGRESS_CLASS=nginx install_k3s
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku apps:create $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains 'from cnb stack'
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command[0]}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "launcher"
|
||||||
|
|
||||||
|
cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
|
||||||
|
run /bin/bash -c "echo $cron_id"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_exists
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "['task.py', 'some', 'cron', 'task']"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "(scheduler-k3s) cnb dokku run uses launcher entrypoint" {
|
||||||
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||||
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
INGRESS_CLASS=nginx install_k3s
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku apps:create $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains 'from cnb stack'
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku --quiet run $TEST_APP python3 task.py test"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "['task.py', 'test']"
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku --quiet run $TEST_APP env"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "SECRET_KEY=fjdkslafjdk"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "(scheduler-k3s) cnb dokku run resolves Procfile key" {
|
||||||
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||||
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
INGRESS_CLASS=nginx install_k3s
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku apps:create $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains 'from cnb stack'
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku --quiet run $TEST_APP task"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "['task.py', 'test']"
|
||||||
|
|
||||||
|
cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
|
||||||
|
run /bin/bash -c "echo $cron_id"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_exists
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "['task.py', 'test']"
|
||||||
|
}
|
||||||
|
|
||||||
|
cron_run_wrapper() {
|
||||||
|
local APP="$1"
|
||||||
|
local APP_REPO_DIR="$2"
|
||||||
|
[[ -z "$APP" ]] && local APP="$TEST_APP"
|
||||||
|
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
|
||||||
|
|
||||||
|
add_requirements_txt "$APP" "$APP_REPO_DIR"
|
||||||
|
mv -f "$APP_REPO_DIR/app-cnb-cron.json" "$APP_REPO_DIR/app.json"
|
||||||
|
}
|
||||||
|
|
||||||
|
cron_run_procfile_wrapper() {
|
||||||
|
local APP="$1"
|
||||||
|
local APP_REPO_DIR="$2"
|
||||||
|
[[ -z "$APP" ]] && local APP="$TEST_APP"
|
||||||
|
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
|
||||||
|
|
||||||
|
add_requirements_txt "$APP" "$APP_REPO_DIR"
|
||||||
|
mv -f "$APP_REPO_DIR/app-cnb-cron-procfile.json" "$APP_REPO_DIR/app.json"
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user