From d72c183d9e4cfa60e20656960ee5c007502d4f21 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 21 Sep 2025 18:00:07 -0400 Subject: [PATCH 01/16] feat: add ability to specify a concurrency policy for crons in the app.json file --- docs/appendices/file-formats/app-json.md | 1 + plugins/app-json/appjson.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docs/appendices/file-formats/app-json.md b/docs/appendices/file-formats/app-json.md index c7500e19f..842674376 100644 --- a/docs/appendices/file-formats/app-json.md +++ b/docs/appendices/file-formats/app-json.md @@ -23,6 +23,7 @@ - `command`: (string, required) - `maintenance`: (boolean, optional) - `schedule`: (string, required) +- `concurrency_policy`: (string, optional, default: `allow`, options: `allow`, `deny`, `replace`) ## Formation diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 99c856bc5..fe59c9942 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -60,6 +60,9 @@ type CronTask struct { // Schedule is the cron schedule to execute the command on Schedule string `json:"schedule"` + + // ConcurrencyPolicy is the concurrency policy for the cron command + ConcurrencyPolicy string `json:"concurrency_policy"` } // Formation is a struct that represents the scale for a process from an app.json file From f1e3c0764c1e7bb25b61b2d6ed574da2b5399afe Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 21 Sep 2025 18:06:21 -0400 Subject: [PATCH 02/16] feat: expose the concurrency policy when fetching cron entries Also add some validation to ensure invalid values are not specified. --- plugins/cron/cron.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/cron/cron.go b/plugins/cron/cron.go index 1d02ade85..aa62ed75c 100644 --- a/plugins/cron/cron.go +++ b/plugins/cron/cron.go @@ -47,6 +47,9 @@ type CronTask struct { // Schedule is the cron schedule Schedule string `json:"schedule"` + // ConcurrencyPolicy is the concurrency policy for the cron command + ConcurrencyPolicy string `json:"concurrency_policy"` + // AltCommand is an alternate command to run AltCommand string `json:"-"` @@ -148,12 +151,19 @@ func FetchCronTasks(input FetchCronTasksInput) ([]CronTask, error) { maintenance = boolValue } } + if c.ConcurrencyPolicy == "" { + c.ConcurrencyPolicy = "allow" + } + if c.ConcurrencyPolicy != "allow" && c.ConcurrencyPolicy != "forbid" && c.ConcurrencyPolicy != "replace" { + return tasks, fmt.Errorf("Invalid cron concurrency policy for app %s (schedule %s): %s", appName, c.Schedule, c.ConcurrencyPolicy) + } tasks = append(tasks, CronTask{ App: appName, Command: c.Command, Schedule: c.Schedule, ID: cronID, + ConcurrencyPolicy: c.ConcurrencyPolicy, Maintenance: isAppCronInMaintenance || maintenance, AppInMaintenance: isAppCronInMaintenance, TaskInMaintenance: maintenance, From f38bc641194ab67b22da077502a95bcae66418cf Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 21 Sep 2025 18:12:25 -0400 Subject: [PATCH 03/16] feat: expose concurrency policy to kubernetes jobs --- plugins/scheduler-k3s/template.go | 17 +++++++++++++---- .../scheduler-k3s/templates/chart/cron-job.yaml | 2 +- plugins/scheduler-k3s/triggers.go | 10 ++++++---- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 17eafdde5..dcc6060d8 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -299,12 +299,21 @@ const ( ) type ProcessCron struct { - ID string `yaml:"id"` - Schedule string `yaml:"schedule"` - Suffix string `yaml:"suffix"` - Suspend bool `yaml:"suspend"` + ID string `yaml:"id"` + Schedule string `yaml:"schedule"` + Suffix string `yaml:"suffix"` + Suspend bool `yaml:"suspend"` + ConcurrencyPolicy ProcessCronConcurrencyPolicy `yaml:"concurrency_policy"` } +type ProcessCronConcurrencyPolicy string + +const ( + ProcessCronConcurrencyPolicy_Allow ProcessCronConcurrencyPolicy = "Allow" + ProcessCronConcurrencyPolicy_Forbid ProcessCronConcurrencyPolicy = "Forbid" + ProcessCronConcurrencyPolicy_Replace ProcessCronConcurrencyPolicy = "Replace" +) + type ProcessPortMap struct { ContainerPort int32 `yaml:"container_port"` HostPort int32 `yaml:"host_port"` diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index b5b4e8b88..102b3ce1e 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -26,7 +26,7 @@ metadata: name: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} namespace: {{ $.Values.global.namespace }} spec: - concurrencyPolicy: Allow + concurrencyPolicy: {{ $config.cron.concurrency_policy }} failedJobsHistoryLimit: 10 jobTemplate: metadata: diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 626f11a5d..1086a7163 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -699,14 +699,16 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting process labels: %w", err) } + concurrencyPolicy := strings.ToUpper(cronTask.ConcurrencyPolicy) processValues := ProcessValues{ Args: words, Annotations: annotations, Cron: ProcessCron{ - ID: cronTask.ID, - Schedule: cronTask.Schedule, - Suffix: suffix, - Suspend: cronTask.Maintenance, + ID: cronTask.ID, + Schedule: cronTask.Schedule, + Suffix: suffix, + Suspend: cronTask.Maintenance, + ConcurrencyPolicy: ProcessCronConcurrencyPolicy(concurrencyPolicy), }, Labels: labels, ProcessType: ProcessType_Cron, From 20be426125d1aa1f52a276177565a98a2eba0929 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 21 Sep 2025 19:10:01 -0400 Subject: [PATCH 04/16] feat: add support for locking in docker-local commands --- plugins/cron/cron.go | 2 +- plugins/run/internal-functions | 17 ++++++++- plugins/scheduler-docker-local/scheduler-run | 39 ++++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/plugins/cron/cron.go b/plugins/cron/cron.go index aa62ed75c..89d19f552 100644 --- a/plugins/cron/cron.go +++ b/plugins/cron/cron.go @@ -75,7 +75,7 @@ func (t CronTask) DokkuRunCommand() string { return t.AltCommand } - return fmt.Sprintf("dokku run --cron-id %s %s %s", t.ID, t.App, t.Command) + return fmt.Sprintf("dokku run --concurrency-policy %s --cron-id %s %s %s", t.ConcurrencyPolicy, t.ID, t.App, t.Command) } // FetchCronTasksInput is the input for the FetchCronTasks function diff --git a/plugins/run/internal-functions b/plugins/run/internal-functions index b4ccfc2ce..fe45b1790 100755 --- a/plugins/run/internal-functions +++ b/plugins/run/internal-functions @@ -9,7 +9,7 @@ fn-run() { shift 1 declare APP="" - local CRON_ID + local CRON_ID CONCURRENCY_POLICY="allow" declare -a RUN_ENV RUN_ENV=() while [[ $# -gt 0 ]]; do @@ -19,6 +19,11 @@ fn-run() { CRON_ID="$arg" shift ;; + --concurrency-policy=*) + local arg=$(printf "%s" "$1" | sed -E 's/(^--concurrency-policy=)//g') + CONCURRENCY_POLICY="$arg" + shift + ;; --no-tty) export DOKKU_DISABLE_TTY=true shift @@ -35,6 +40,14 @@ fn-run() { CRON_ID="$2" shift 2 ;; + --concurrency-policy) + if [[ ! $2 ]]; then + dokku_log_warn "expected $1 to have an argument" + break + fi + CONCURRENCY_POLICY="$2" + shift 2 + ;; -e=* | --env=*) local arg=$(printf "%s" "$1" | sed -E 's/(^-e=)|(^--env=)//g') RUN_ENV+=("$arg") @@ -67,7 +80,7 @@ fn-run() { verify_app_name "$APP" local DOKKU_SCHEDULER=$(get_app_scheduler "$APP") - DOKKU_CRON_ID="$CRON_ID" plugn trigger scheduler-run "$DOKKU_SCHEDULER" "$APP" "${#RUN_ENV[@]}" "${RUN_ENV[@]}" -- "$@" + DOKKU_CRON_ID="$CRON_ID" DOKKU_CONCURRENCY_POLICY="$CONCURRENCY_POLICY" plugn trigger scheduler-run "$DOKKU_SCHEDULER" "$APP" "${#RUN_ENV[@]}" "${RUN_ENV[@]}" -- "$@" } cmd-run() { diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index cb6bbfb74..096210dc3 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -5,6 +5,25 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions" +fn-scheduler-docker-local-scheduler-run-acquire-lock() { + declare desc="acquire advisory lock for use in docker-local cron jobs" + local APP="$1" CRON_ID="$2" + local LOCK_TYPE="exclusive" + local APP_DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/scheduler-docker-local/$APP/cron-$CRON_ID.lock" + local LOCK_WAITING_MSG="$APP currently has a cron lock in place for $CRON_ID. Waiting..." + local LOCK_FAILED_MSG="$APP currently has a cron lock in place for $CRON_ID. Exiting..." + + acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG" +} + +fn-scheduler-docker-local-scheduler-run-release-lock() { + declare desc="release advisory lock used in docker-local cron jobs" + local APP="$1" CRON_ID="$2" + local APP_DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/scheduler-docker-local/$APP/cron-$CRON_ID.lock" + + release_advisory_lock "$APP_DEPLOY_LOCK_FILE" +} + trigger-scheduler-docker-local-scheduler-run() { declare desc="runs command in container based on app image" declare trigger="scheduler-run" @@ -39,9 +58,24 @@ trigger-scheduler-docker-local-scheduler-run() { [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" -e TRACE=true " local PROCESS_TYPE=run + local concurrency_policy="allow" if [[ -n "$DOKKU_CRON_ID" ]]; then PROCESS_TYPE=cron DOCKER_ARGS+=" --label=com.dokku.cron-id=$DOKKU_CRON_ID" + + if [[ -n "$DOKKU_CONCURRENCY_POLICY" ]]; then + concurrency_policy="$DOKKU_CONCURRENCY_POLICY" + fi + concurrency_policy="$DOKKU_CONCURRENCY_POLICY" + DOCKER_ARGS+=" --label=com.dokku.concurrency-policy=$concurrency_policy" + + if [[ "$concurrency_policy" == "forbid" ]]; then + fn-scheduler-docker-local-scheduler-run-acquire-lock "$APP" "$DOKKU_CRON_ID" + fi + + if [[ "$concurrency_policy" == "replace" ]]; then + dokku_log_warn "Concurrency policy 'replace' is not supported for docker-local cron jobs" + fi fi local DYNO_NUMBER="$RANDOM" @@ -138,6 +172,11 @@ trigger-scheduler-docker-local-scheduler-run() { DOKKU_CONTAINER_EXIT_CODE="$("$DOCKER_BIN" container wait "$CONTAINER_ID" 2>/dev/null || echo "$EXIT_CODE")" [[ -z "$DOKKU_CONTAINER_EXIT_CODE" ]] && DOKKU_CONTAINER_EXIT_CODE=0 fi + + if [[ -n "$DOKKU_CRON_ID" ]] && [[ "$concurrency_policy" == "forbid" ]]; then + fn-scheduler-docker-local-scheduler-run-release-lock "$APP" "$DOKKU_CRON_ID" + fi + plugn trigger scheduler-post-run "$DOKKU_SCHEDULER" "$APP" "$CONTAINER_ID" "$DOKKU_CONTAINER_EXIT_CODE" return "$DOKKU_CONTAINER_EXIT_CODE" } From b8f32f6ea42d9b71dba53e1dce32563976f19e8e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 5 Oct 2025 15:45:47 -0400 Subject: [PATCH 05/16] tests: add concurrency_policy to test check --- tests/unit/cron.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cron.bats b/tests/unit/cron.bats index 5c0b1345d..671e8f58f 100644 --- a/tests/unit/cron.bats +++ b/tests/unit/cron.bats @@ -129,7 +129,7 @@ teardown() { echo "output: $output" echo "status: $status" assert_success - assert_output '[{"id":"5cruaotm4yzzpnjlsdunblj8qyjp","command":"/bin/true","global":true,"schedule":"@daily","app-in-maintenance":false,"task-in-maintenance":false,"maintenance":false}]' + assert_output '[{"id":"5cruaotm4yzzpnjlsdunblj8qyjp","command":"/bin/true","global":true,"schedule":"@daily","concurrency_policy":"","app-in-maintenance":false,"task-in-maintenance":false,"maintenance":false}]' run /bin/bash -c "cat /var/spool/cron/crontabs/dokku" echo "output: $output" From 983115c8277979b7045aefa74b122369f57e914c Mon Sep 17 00:00:00 2001 From: Corey Taylor Date: Tue, 21 Oct 2025 21:44:11 -0500 Subject: [PATCH 06/16] Add test for cron concurrency policy --- docs/appendices/file-formats/app-json.md | 2 +- plugins/cron/subcommands.go | 4 +- plugins/scheduler-docker-local/scheduler-run | 1 - tests/unit/cron.bats | 42 ++++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/docs/appendices/file-formats/app-json.md b/docs/appendices/file-formats/app-json.md index 842674376..b3cd72c7b 100644 --- a/docs/appendices/file-formats/app-json.md +++ b/docs/appendices/file-formats/app-json.md @@ -23,7 +23,7 @@ - `command`: (string, required) - `maintenance`: (boolean, optional) - `schedule`: (string, required) -- `concurrency_policy`: (string, optional, default: `allow`, options: `allow`, `deny`, `replace`) +- `concurrency_policy`: (string, optional, default: `allow`, options: `allow`, `forbid`, `replace`) ## Formation diff --git a/plugins/cron/subcommands.go b/plugins/cron/subcommands.go index 233361953..c47fa3289 100644 --- a/plugins/cron/subcommands.go +++ b/plugins/cron/subcommands.go @@ -42,7 +42,7 @@ func CommandList(appName string, format string) error { } if format == "stdout" { - output := []string{"ID | Schedule | Maintenance | Command"} + output := []string{"ID | Schedule | Concurrency | Maintenance | Command"} for _, task := range tasks { maintenance := "false" if task.Maintenance { @@ -52,7 +52,7 @@ func CommandList(appName string, format string) error { maintenance = "true (app)" } } - output = append(output, fmt.Sprintf("%s | %s | %s | %s", task.ID, task.Schedule, maintenance, task.Command)) + output = append(output, fmt.Sprintf("%s | %s | %s | %t | %s", task.ID, task.Schedule, task.ConcurrencyPolicy, maintenance, task.Command)) } result := columnize.SimpleFormat(output) diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index 096210dc3..0df116077 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -66,7 +66,6 @@ trigger-scheduler-docker-local-scheduler-run() { if [[ -n "$DOKKU_CONCURRENCY_POLICY" ]]; then concurrency_policy="$DOKKU_CONCURRENCY_POLICY" fi - concurrency_policy="$DOKKU_CONCURRENCY_POLICY" DOCKER_ARGS+=" --label=com.dokku.concurrency-policy=$concurrency_policy" if [[ "$concurrency_policy" == "forbid" ]]; then diff --git a/tests/unit/cron.bats b/tests/unit/cron.bats index 671e8f58f..fa5d8a0e4 100644 --- a/tests/unit/cron.bats +++ b/tests/unit/cron.bats @@ -235,6 +235,30 @@ teardown() { assert_output "['task.py', 'schedule', 'now']" } +@test "(cron) cron:run concurrency_policy forbid" { + run deploy_app dockerfile dokku@$DOKKU_DOMAIN:$TEST_APP template_cron_file_concurrency_forbid + echo "output: $output" + echo "status: $status" + assert_success + + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" + run /bin/bash -c "echo cron $cron_id" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + + run /bin/bash -c "dokku cron:run $TEST_APP $cron_id --detach" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku cron:run $TEST_APP $cron_id" + echo "output: $output" + echo "status: $status" + assert_failure +} + @test "(cron) cron:suspend cron:resume" { run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP template_cron_file_valid_multiple echo "output: $output" @@ -427,3 +451,21 @@ template_cron_file_valid_multiple() { } EOF } + +template_cron_file_concurrency_forbid() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + echo "injecting valid cron app.json -> $APP_REPO_DIR/app.json" + cat <"$APP_REPO_DIR/app.json" +{ + "cron": [ + { + "command": "sleep 5", + "schedule": "0 0 * * *", + "concurrency_policy": "forbid" + } + ] +} +EOF +} From c95cb526bed17b45c71a16a6d4db345c4d864054 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 27 Oct 2025 14:28:18 -0400 Subject: [PATCH 07/16] fix: correct lint issue --- tests/unit/cron.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cron.bats b/tests/unit/cron.bats index fa5d8a0e4..6da29182d 100644 --- a/tests/unit/cron.bats +++ b/tests/unit/cron.bats @@ -240,7 +240,7 @@ teardown() { echo "output: $output" echo "status: $status" assert_success - + cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')" run /bin/bash -c "echo cron $cron_id" echo "output: $output" From 8c2731b328683a7492a4aefbeab7a90679d97064 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 27 Oct 2025 14:37:13 -0400 Subject: [PATCH 08/16] fix: properly set concurrency policy for golang --- plugins/scheduler-k3s/triggers.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 1086a7163..7ae406ee4 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -700,6 +700,16 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } concurrencyPolicy := strings.ToUpper(cronTask.ConcurrencyPolicy) + switch concurrencyPolicy { + case "ALLOW": + concurrencyPolicy = "Allow" + case "FORBID": + concurrencyPolicy = "Forbid" + case "REPLACE": + concurrencyPolicy = "Replace" + default: + return fmt.Errorf("Invalid concurrency_policy specified: %v", concurrencyPolicy) + } processValues := ProcessValues{ Args: words, Annotations: annotations, From 3162fa5b9a64f342c95aa11f39da477d345eed94 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 10 Nov 2025 18:02:46 -0500 Subject: [PATCH 09/16] debug: run sleep for longer and check the inspect output of the cron container --- tests/unit/cron.bats | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit/cron.bats b/tests/unit/cron.bats index 6da29182d..8b37cac2c 100644 --- a/tests/unit/cron.bats +++ b/tests/unit/cron.bats @@ -253,6 +253,12 @@ teardown() { echo "status: $status" assert_success + run /bin/bash -c "docker ps --filter "label=com.dokku.cron-id=$cron_id" -q | xargs docker inspect" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_exists + run /bin/bash -c "dokku cron:run $TEST_APP $cron_id" echo "output: $output" echo "status: $status" @@ -461,7 +467,7 @@ template_cron_file_concurrency_forbid() { { "cron": [ { - "command": "sleep 5", + "command": "sleep 30", "schedule": "0 0 * * *", "concurrency_policy": "forbid" } From c56110b3a72ad600c421af701c3c17ff4938108f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 13 Nov 2025 01:52:08 -0500 Subject: [PATCH 10/16] fix: do not print errors twice --- plugins/common/log.go | 4 +++- plugins/cron/subcommands.go | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/common/log.go b/plugins/common/log.go index 9a3281138..9e67287b4 100644 --- a/plugins/common/log.go +++ b/plugins/common/log.go @@ -90,7 +90,9 @@ func LogFailWithError(err error) { fmt.Fprintf(os.Stderr, " ! %s\n", e.Error()) } } else { - fmt.Fprintf(os.Stderr, " ! %s\n", err.Error()) + if err.Error() != "" { + fmt.Fprintf(os.Stderr, " ! %s\n", err.Error()) + } } if errExit, ok := err.(ErrWithExitCode); ok { os.Exit(errExit.ExitCode()) diff --git a/plugins/cron/subcommands.go b/plugins/cron/subcommands.go index c47fa3289..c46ba4771 100644 --- a/plugins/cron/subcommands.go +++ b/plugins/cron/subcommands.go @@ -143,6 +143,11 @@ func CommandRun(appName string, cronID string, detached bool) error { Args: args, StreamStdio: true, }) + if err != nil { + // return an error with an empty message to avoid + // printing the error message twice + return errors.New("") + } return err } From 3f8c868ab856eeee82c1e9d433694e3f205c45cb Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 13 Nov 2025 01:53:33 -0500 Subject: [PATCH 11/16] fix: ensure the concurrency policy is propagated when running a cron command directly --- plugins/cron/subcommands.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/cron/subcommands.go b/plugins/cron/subcommands.go index c46ba4771..8bc86677f 100644 --- a/plugins/cron/subcommands.go +++ b/plugins/cron/subcommands.go @@ -112,9 +112,11 @@ func CommandRun(appName string, cronID string, detached bool) error { } command := "" + concurrencyPolicy := "allow" for _, task := range tasks { if task.ID == cronID { command = task.Command + concurrencyPolicy = task.ConcurrencyPolicy } } @@ -134,6 +136,7 @@ func CommandRun(appName string, cronID string, detached bool) error { os.Setenv("DOKKU_DISABLE_TTY", "true") } + os.Setenv("DOKKU_CONCURRENCY_POLICY", concurrencyPolicy) os.Setenv("DOKKU_CRON_ID", cronID) os.Setenv("DOKKU_RM_CONTAINER", "1") scheduler := common.GetAppScheduler(appName) From bacbc6d90d4c2055235f5a4e2fbaef3ab0de22b2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 13 Nov 2025 01:54:47 -0500 Subject: [PATCH 12/16] fix: correctly check for running containers when concurrency policy is forbid We cannot grab a lock as cron:run with --detach will release the lock. Instead, just check if any containers for the specified cron_id are running and exit. --- plugins/scheduler-docker-local/scheduler-run | 30 ++++---------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index 0df116077..3ff3077b6 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -5,25 +5,6 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions" -fn-scheduler-docker-local-scheduler-run-acquire-lock() { - declare desc="acquire advisory lock for use in docker-local cron jobs" - local APP="$1" CRON_ID="$2" - local LOCK_TYPE="exclusive" - local APP_DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/scheduler-docker-local/$APP/cron-$CRON_ID.lock" - local LOCK_WAITING_MSG="$APP currently has a cron lock in place for $CRON_ID. Waiting..." - local LOCK_FAILED_MSG="$APP currently has a cron lock in place for $CRON_ID. Exiting..." - - acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG" -} - -fn-scheduler-docker-local-scheduler-run-release-lock() { - declare desc="release advisory lock used in docker-local cron jobs" - local APP="$1" CRON_ID="$2" - local APP_DEPLOY_LOCK_FILE="$DOKKU_LIB_ROOT/data/scheduler-docker-local/$APP/cron-$CRON_ID.lock" - - release_advisory_lock "$APP_DEPLOY_LOCK_FILE" -} - trigger-scheduler-docker-local-scheduler-run() { declare desc="runs command in container based on app image" declare trigger="scheduler-run" @@ -69,7 +50,12 @@ trigger-scheduler-docker-local-scheduler-run() { DOCKER_ARGS+=" --label=com.dokku.concurrency-policy=$concurrency_policy" if [[ "$concurrency_policy" == "forbid" ]]; then - fn-scheduler-docker-local-scheduler-run-acquire-lock "$APP" "$DOKKU_CRON_ID" + # check if there is a running docker container with the same com.dokku.cron-id label + local RUNNING_CONTAINERS="$("$DOCKER_BIN" container ls --filter "label=com.dokku.cron-id=$DOKKU_CRON_ID" --filter "status=running" --quiet || true)" + if [[ -n "$RUNNING_CONTAINERS" ]]; then + dokku_log_warn "$APP currently has a cron lock in place for $DOKKU_CRON_ID. Exiting..." + return 1 + fi fi if [[ "$concurrency_policy" == "replace" ]]; then @@ -172,10 +158,6 @@ trigger-scheduler-docker-local-scheduler-run() { [[ -z "$DOKKU_CONTAINER_EXIT_CODE" ]] && DOKKU_CONTAINER_EXIT_CODE=0 fi - if [[ -n "$DOKKU_CRON_ID" ]] && [[ "$concurrency_policy" == "forbid" ]]; then - fn-scheduler-docker-local-scheduler-run-release-lock "$APP" "$DOKKU_CRON_ID" - fi - plugn trigger scheduler-post-run "$DOKKU_SCHEDULER" "$APP" "$CONTAINER_ID" "$DOKKU_CONTAINER_EXIT_CODE" return "$DOKKU_CONTAINER_EXIT_CODE" } From 42a92eb6549f4ee5d6520fb6a3f3a8c6e7b2e86d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 13 Nov 2025 01:55:12 -0500 Subject: [PATCH 13/16] feat: add support for replace concurrency policy --- plugins/scheduler-docker-local/scheduler-run | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index 3ff3077b6..b83510684 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -59,7 +59,13 @@ trigger-scheduler-docker-local-scheduler-run() { fi if [[ "$concurrency_policy" == "replace" ]]; then - dokku_log_warn "Concurrency policy 'replace' is not supported for docker-local cron jobs" + local RUNNING_CONTAINERS="$("$DOCKER_BIN" container ls --filter "label=com.dokku.cron-id=$DOKKU_CRON_ID" --filter "status=running" --quiet || true)" + if [[ -n "$RUNNING_CONTAINERS" ]]; then + dokku_log_warn "Replacing running container for $DOKKU_CRON_ID" + "$DOCKER_BIN" container update --restart=no "$RUNNING_CONTAINERS" &>/dev/null || true + "$DOCKER_BIN" container stop "$RUNNING_CONTAINERS" >/dev/null && "$DOCKER_BIN" container kill "$RUNNING_CONTAINERS" &>/dev/null + plugn trigger scheduler-register-retired "$APP" "$RUNNING_CONTAINERS" + fi fi fi From c6bd80aca68169555d412f37086d678ba4b67e5c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 13 Nov 2025 01:55:32 -0500 Subject: [PATCH 14/16] tests: update tests --- tests/unit/cron.bats | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/cron.bats b/tests/unit/cron.bats index 8b37cac2c..5eb2bc34a 100644 --- a/tests/unit/cron.bats +++ b/tests/unit/cron.bats @@ -253,15 +253,16 @@ teardown() { echo "status: $status" assert_success - run /bin/bash -c "docker ps --filter "label=com.dokku.cron-id=$cron_id" -q | xargs docker inspect" + run /bin/bash -c "docker ps --filter "label=com.dokku.cron-id=$cron_id" -q | xargs docker inspect -f '{{ index .Config.Labels \"com.dokku.concurrency-policy\" }}'" echo "output: $output" echo "status: $status" assert_success - assert_output_exists + assert_output "forbid" run /bin/bash -c "dokku cron:run $TEST_APP $cron_id" echo "output: $output" echo "status: $status" + assert_output_contains "currently has a cron lock in place for $cron_id" assert_failure } From dc8424d8a52b911c4d87746cb90a13286a7ebd24 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 13 Nov 2025 01:58:37 -0500 Subject: [PATCH 15/16] docs: document the concurrency_policy --- docs/processes/scheduled-cron-tasks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/processes/scheduled-cron-tasks.md b/docs/processes/scheduled-cron-tasks.md index 39f659586..6ce3b5f71 100644 --- a/docs/processes/scheduled-cron-tasks.md +++ b/docs/processes/scheduled-cron-tasks.md @@ -38,6 +38,8 @@ A cron task takes the following properties: - `command`: A command to be run within the built app image. Specified commands can also be `Procfile` entries. - `maintenance`: A boolean value that decides whether the cron task is in maintenance and therefore executable or not. - `schedule`: A [cron-compatible](https://en.wikipedia.org/wiki/Cron#Overview) scheduling definition upon which to run the command. Seconds are generally not supported. +- `concurrency_policy`: A string (default: `allow`), that controls whether the cron task can be run concurrently with another invocation of itself. Valid options are `allow` (allow concurrency), `forbid` (exit the new cron task if there is an existing one), `replace` (delete any existing cron task and start the new one). + Zero or more cron tasks can be specified per app. Cron tasks are validated after the build artifact is created but before the app is deployed, and the cron schedule is updated during the post-deploy phase. From 3f707284515088b77ee34bf7cff7afb8510ebe4c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 15 Nov 2025 16:19:15 -0500 Subject: [PATCH 16/16] feat: implement forbid/replace logic for k3s pods spawned via cron:run --- plugins/scheduler-k3s/k8s.go | 45 +++++++++++++++++++++++++++++++ plugins/scheduler-k3s/triggers.go | 44 +++++++++++++++++++++++------- 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index aa5a07543..14ee6002b 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -337,6 +337,51 @@ func (k KubernetesClient) DeleteNode(ctx context.Context, input DeleteNodeInput) return k.Client.CoreV1().Nodes().Delete(ctx, input.Name, metav1.DeleteOptions{}) } +// DeletePodInput contains all the information needed to delete a Kubernetes pod +type DeletePodInput struct { + // Name is the Kubernetes pod name + Name string + + // Namespace is the Kubernetes namespace + Namespace string + + // LabelSelector is the Kubernetes label selector + LabelSelector string +} + +// DeletePod deletes a Kubernetes pod +func (k KubernetesClient) DeletePod(ctx context.Context, input DeletePodInput) error { + if input.Name == "" && input.LabelSelector == "" { + return fmt.Errorf("name or label selector is required") + } + + if input.Name != "" && input.LabelSelector != "" { + return fmt.Errorf("name and label selector cannot be used together") + } + + if input.Name != "" { + return k.Client.CoreV1().Pods(input.Namespace).Delete(ctx, input.Name, metav1.DeleteOptions{}) + } + + // get the pods with the label selector + pods, err := k.ListPods(ctx, ListPodsInput{ + Namespace: input.Namespace, + LabelSelector: input.LabelSelector, + }) + if err != nil { + return fmt.Errorf("Error listing pods: %w", err) + } + + for _, pod := range pods { + err := k.Client.CoreV1().Pods(input.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("Error deleting pod %s: %w", pod.Name, err) + } + } + + return nil +} + // DeleteSecretInput contains all the information needed to delete a Kubernetes secret type DeleteSecretInput struct { // Name is the Kubernetes secret name diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 7ae406ee4..de2e9c2c9 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -1212,10 +1212,44 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] extraEnv["TRACE"] = "true" } + namespace := getComputedNamespace(appName) + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Error creating kubernetes client: %w", err) + } + + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + processType := "run" if os.Getenv("DOKKU_CRON_ID") != "" { processType = "cron" labels["dokku.com/cron-id"] = os.Getenv("DOKKU_CRON_ID") + concurrencyPolicy := strings.ToUpper(os.Getenv("DOKKU_CONCURRENCY_POLICY")) + switch concurrencyPolicy { + case "forbid": + // check if there is a running pod with the same dokku.com/cron-id label + pods, err := clientset.ListPods(context.Background(), ListPodsInput{ + Namespace: namespace, + LabelSelector: fmt.Sprintf("dokku.com/cron-id=%s", os.Getenv("DOKKU_CRON_ID")), + }) + if err != nil { + return fmt.Errorf("Error listing pods: %w", err) + } + if len(pods) > 0 { + return fmt.Errorf("There is a running pod with the same dokku.com/cron-id label") + } + case "replace": + // delete any existing pod with the same dokku.com/cron-id label + err := clientset.DeletePod(context.Background(), DeletePodInput{ + Namespace: namespace, + LabelSelector: fmt.Sprintf("dokku.com/cron-id=%s", os.Getenv("DOKKU_CRON_ID")), + }) + if err != nil { + return fmt.Errorf("Error deleting pod: %w", err) + } + } } imageSourceType, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.builder-type\" }}") @@ -1245,7 +1279,6 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] entrypoint = "/exec" } - namespace := getComputedNamespace(appName) helmAgent, err := NewHelmAgent(namespace, DevNullPrinter) if err != nil { return fmt.Errorf("Error creating helm agent: %w", err) @@ -1275,15 +1308,6 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] attachToPod := os.Getenv("DOKKU_DETACH_CONTAINER") != "1" - clientset, err := NewKubernetesClient() - if err != nil { - return fmt.Errorf("Error creating kubernetes client: %w", err) - } - - if err := clientset.Ping(); err != nil { - return fmt.Errorf("kubernetes api not available: %w", err) - } - imagePullSecrets := getComputedImagePullSecrets(appName) if imagePullSecrets == "" { imagePullSecrets = fmt.Sprintf("ims-%s.%d", appName, deploymentID)