diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 1d202951c..77788ad3b 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -2805,6 +2805,26 @@ DOKKU_SCHEDULER="$1"; APP="$2"; CONTAINER="$3"; TAIL="$4"; PRETTY_PRINT="$5"; NU # TODO ``` +### `scheduler-run-retire` + +> [!WARNING] +> The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + +- Description: Allows you to retire containers started by the `run` command +- Invoked by: `dokku run:retire` +- Arguments: `$DOKKU_SCHEDULER $APP` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +DOKKU_SCHEDULER="$1"; APP="$2"; + +# TODO +``` + ### `scheduler-run-stop` > [!WARNING] diff --git a/docs/processes/one-off-tasks.md b/docs/processes/one-off-tasks.md index 1df42652c..f3f699111 100644 --- a/docs/processes/one-off-tasks.md +++ b/docs/processes/one-off-tasks.md @@ -1,8 +1,8 @@ # One-off Tasks ``` -run [-e|--env KEY=VALUE] [--no-tty] # Run a command in a new container using the current app image -run:detached [-e|-env KEY=VALUE] [--force-tty] # Run a command in a new detached container using the current app image +run [-e|--env KEY=VALUE] [--no-tty] [--ttl-seconds SECONDS] # Run a command in a new container using the current app image +run:detached [-e|-env KEY=VALUE] [--force-tty] [--ttl-seconds SECONDS] # Run a command in a new detached container using the current app image run:list [--format json|stdout] [] # List all run containers for an app run:logs [-h] [-t] [-n num] [-q] # Display recent log output for run containers run:stop # Stops all run containers for an app or a specified run container @@ -29,6 +29,16 @@ dokku run --env "NODE_ENV=development" --env "PATH=/custom/path" node-js-app npm One off containers are removed at the end of process execution. +In addition, one-off containers also run for a max of 24 hours (86400 seconds) by default, after which they are reaped. This can be changed by setting the `--ttl-seconds` argument: + + +```shell +# runs for 10 minutes +dokku run --ttl-seconds 600 node-js-app npm run mytask +``` + +One-off containers that exceed their runtime are reaped every 5 minutes, so the ttl-seconds is an approximation and your app may end up running for up to 5 minutes longer than expected. + #### Running Procfile commands The `run` command can also be used to run a command defined in the app `Procfile`: diff --git a/docs/processes/scheduled-cron-tasks.md b/docs/processes/scheduled-cron-tasks.md index 6ce3b5f71..e9473312d 100644 --- a/docs/processes/scheduled-cron-tasks.md +++ b/docs/processes/scheduled-cron-tasks.md @@ -43,6 +43,8 @@ A cron task takes the following properties: Zero or more cron tasks can be specified per app. Cron tasks are validated after the build artifact is created but before the app is deployed, and the cron schedule is updated during the post-deploy phase. +Cron tasks can run for a maximum of 24 hours via the docker-local scheduler, after which they are reaped from the system. + See the [app.json location documentation](/docs/advanced-usage/deployment-tasks.md#changing-the-appjson-location) for more information on where to place your `app.json` file. #### Task Environment diff --git a/plugins/git/git-from-directory b/plugins/git/git-from-directory index 771a7bba9..238fb297b 100755 --- a/plugins/git/git-from-directory +++ b/plugins/git/git-from-directory @@ -18,7 +18,7 @@ trigger-git-git-from-directory() { trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT local has_code=false - if fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" >/dev/null 2>&1; then + if fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then has_code=true fi diff --git a/plugins/git/internal-functions b/plugins/git/internal-functions index 0cd956750..5641cd923 100755 --- a/plugins/git/internal-functions +++ b/plugins/git/internal-functions @@ -211,7 +211,7 @@ cmd-git-sync() { DOKKU_DEPLOY_BRANCH="$(fn-git-deploy-branch "$APP")" CURRENT_REF="$(fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" 2>/dev/null || true)" - if ! fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" >/dev/null 2>&1; then + if ! fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then dokku_log_info1_quiet "Cloning $APP from $GIT_REMOTE#$GIT_REF" fn-git-clone "$APP" "$GIT_REMOTE" "$GIT_REF" else @@ -378,7 +378,7 @@ fn-git-clone() { [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" local DOKKU_DEPLOY_BRANCH="$(fn-git-deploy-branch "$APP")" - if fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" >/dev/null 2>&1; then + if fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then dokku_log_fail "The clone subcommand can only be executed for new applications" fi @@ -441,7 +441,7 @@ fn-git-fetch() { local APP_ROOT="$DOKKU_ROOT/$APP" local DOKKU_DEPLOY_BRANCH="$(fn-git-deploy-branch "$APP")" - if ! fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" >/dev/null 2>&1; then + if ! fn-git-cmd "$APP_ROOT" rev-parse "$DOKKU_DEPLOY_BRANCH" &>/dev/null; then dokku_log_fail "The fetch subcommand can only be executed for existing applications" fi diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 783a26161..c95f05d74 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -142,6 +142,19 @@ func CommandRetire(appName string) error { Args: []string{scheduler, appName}, StreamStdio: true, }) + if err != nil { + return fmt.Errorf("Error retiring containers: %w", err) + } + + common.LogInfo1("Retiring expired run containers") + _, err = common.CallPlugnTrigger(common.PlugnTriggerInput{ + Trigger: "scheduler-run-retire", + StreamStdio: true, + }) + if err != nil { + return fmt.Errorf("Error retiring expired run containers: %w", err) + } + return err } diff --git a/plugins/run/help-functions b/plugins/run/help-functions index d145031ae..6db9b0eeb 100755 --- a/plugins/run/help-functions +++ b/plugins/run/help-functions @@ -27,10 +27,11 @@ help_desc fn-help-content() { declare desc="return help content" cat < , Run a command in a new container using the current app image - run:detached [-e|-env KEY=VALUE] [--force-tty] , Run a command in a new detached container using the current app image + run [-e|--env KEY=VALUE] [--no-tty] [--ttl-seconds SECONDS] , Run a command in a new container using the current app image + run:detached [-e|-env KEY=VALUE] [--force-tty] [--ttl-seconds SECONDS] , Run a command in a new detached container using the current app image run:list [--format json|stdout] , List all run containers for an app run:logs [-h] [-t] [-n num] [-q], Display recent log output for run containers + run:retire [], Stop all run containers for an app that have exceeded their active deadline run:stop , Stops all run containers for an app or a specified run container help_content } diff --git a/plugins/run/internal-functions b/plugins/run/internal-functions index fe45b1790..e23ada445 100755 --- a/plugins/run/internal-functions +++ b/plugins/run/internal-functions @@ -9,7 +9,7 @@ fn-run() { shift 1 declare APP="" - local CRON_ID CONCURRENCY_POLICY="allow" + local CRON_ID CONCURRENCY_POLICY="allow" DOKKU_RUN_TTL_SECONDS="86400" declare -a RUN_ENV RUN_ENV=() while [[ $# -gt 0 ]]; do @@ -61,6 +61,19 @@ fn-run() { RUN_ENV+=("$2") shift 2 ;; + --ttl-seconds=*) + local arg=$(printf "%s" "$1" | sed -E 's/(^--ttl-seconds=)//g') + DOKKU_RUN_TTL_SECONDS="$arg" + shift + ;; + --ttl-seconds) + if [[ ! $2 ]]; then + dokku_log_warn "expected $1 to have an argument" + break + fi + DOKKU_RUN_TTL_SECONDS="$2" + shift 2 + ;; *) APP="$1" shift @@ -69,6 +82,10 @@ fn-run() { esac done + if [[ -z "$DOKKU_RUN_TTL_SECONDS" ]]; then + DOKKU_RUN_TTL_SECONDS="86400" + fi + if [[ "$CMD" == "run:detached" ]] && [[ "$DOKKU_FORCE_TTY" != "true" ]]; then export DOKKU_DISABLE_TTY=true fi @@ -80,7 +97,7 @@ fn-run() { verify_app_name "$APP" local DOKKU_SCHEDULER=$(get_app_scheduler "$APP") - DOKKU_CRON_ID="$CRON_ID" DOKKU_CONCURRENCY_POLICY="$CONCURRENCY_POLICY" plugn trigger scheduler-run "$DOKKU_SCHEDULER" "$APP" "${#RUN_ENV[@]}" "${RUN_ENV[@]}" -- "$@" + DOKKU_CRON_ID="$CRON_ID" DOKKU_CONCURRENCY_POLICY="$CONCURRENCY_POLICY" DOKKU_RUN_TTL_SECONDS="$DOKKU_RUN_TTL_SECONDS" plugn trigger scheduler-run "$DOKKU_SCHEDULER" "$APP" "${#RUN_ENV[@]}" "${RUN_ENV[@]}" -- "$@" } cmd-run() { @@ -124,6 +141,10 @@ cmd-run-list() { FORMAT="$2" shift 2 ;; + --quiet) + export DOKKU_QUIET_OUTPUT=true + shift + ;; *) APP="$1" shift @@ -201,8 +222,23 @@ cmd-run-logs() { plugn trigger scheduler-run-logs "$DOKKU_SCHEDULER" "$APP" "$CONTAINER_NAME" "$TAIL" "$PRETTY_PRINT" "$NUM" } +cmd-run-retire() { + declare desc="stop all run containers that have exceeded their active deadline" + declare cmd="run:retire" + [[ "$1" == "$cmd" ]] && shift 1 + declare APP="$1" + + if [[ -z "$APP" ]]; then + plugn trigger scheduler-run-retire + else + verify_app_name "$APP" + local DOKKU_SCHEDULER="$(get_app_scheduler "$APP")" + plugn trigger scheduler-run-retire "$DOKKU_SCHEDULER" "$APP" + fi +} + cmd-run-stop() { - declare desc="Stops all run containers for an app or a specified run container" + declare desc="stops all run containers for an app or a specified run container" declare cmd="run:stop" [[ "$1" == "$cmd" ]] && shift 1 declare APP CONTAINER_NAME diff --git a/plugins/run/subcommands/retire b/plugins/run/subcommands/retire new file mode 100755 index 000000000..31d9953f0 --- /dev/null +++ b/plugins/run/subcommands/retire @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eo pipefail +source "$PLUGIN_AVAILABLE_PATH/run/internal-functions" +[[ $DOKKU_TRACE ]] && set -x + +cmd-run-retire "$@" diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index b6c41510a..4586db936 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -119,6 +119,10 @@ trigger-scheduler-docker-local-scheduler-run() { DOCKER_ARGS+=" --platform=linux/amd64" fi + if [[ -n "$DOKKU_RUN_TTL_SECONDS" ]]; then + DOCKER_ARGS+=" --label=com.dokku.active-deadline-seconds=$DOKKU_RUN_TTL_SECONDS" + fi + # shellcheck disable=SC2124 DOCKER_ARGS+=" --label=com.dokku.container-type=$PROCESS_TYPE" DOCKER_ARGS+=" --label=com.dokku.app-name=$APP" diff --git a/plugins/scheduler-docker-local/scheduler-run-list b/plugins/scheduler-docker-local/scheduler-run-list index 2f4ad57b3..93c7cc229 100755 --- a/plugins/scheduler-docker-local/scheduler-run-list +++ b/plugins/scheduler-docker-local/scheduler-run-list @@ -15,7 +15,11 @@ trigger-scheduler-docker-local-scheduler-run-list() { if [[ "$FORMAT" == "stdout" ]]; then dokku_log_info2_quiet "$APP run containers" - "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "table {{.Names}}\t{{.Command}}\t{{.RunningFor}}" + if [[ -n "$DOKKU_QUIET_OUTPUT" ]]; then + "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "{{.Names}}\t{{.Command}}\t{{.RunningFor}}" + else + "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "table {{.Names}}\t{{.Command}}\t{{.RunningFor}}" + fi else "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "{{ json . }}" | jq -s -M 'map({name: .Names, state:.State, command:.Command, created_at:.CreatedAt})' fi diff --git a/plugins/scheduler-docker-local/scheduler-run-retire b/plugins/scheduler-docker-local/scheduler-run-retire new file mode 100755 index 000000000..fe6d3b5f9 --- /dev/null +++ b/plugins/scheduler-docker-local/scheduler-run-retire @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_AVAILABLE_PATH/config/functions" +source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions" + +fn-scheduler-docker-local-run-retire-container() { + declare desc="stop a container" + declare CONTAINER_ID="$1" + + local DOKKU_DOCKER_STOP_TIMEOUT="$(plugn trigger ps-get-property "$APP" stop-timeout-seconds || true)" + + [[ -n "$DOKKU_DOCKER_STOP_TIMEOUT" ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}" + + "$DOCKER_BIN" container update --restart=no "$CONTAINER_ID" &>/dev/null || true + "$DOCKER_BIN" container stop $DOCKER_STOP_TIME_ARG "$CONTAINER_ID" &>/dev/null || true + "$DOCKER_BIN" container kill "$CONTAINER_ID" &>/dev/null || true + "$DOCKER_BIN" container rm "$CONTAINER_ID" &>/dev/null || true + + if "$DOCKER_BIN" container inspect "$CONTAINER_ID" &>/dev/null; then + dokku_log_warn "Unable to retire container ${CONTAINER_ID}" + return 1 + fi +} + +fn-scheduler-docker-local-run-retire() { + declare desc="stop all run containers that have exceeded their active deadline" + declare APP="$1" + local containers="" + + # find all run containers for the specified app + container_type_filter="label=com.dokku.container-type=run" + if [[ -n "$APP" ]]; then + app_filter="label=com.dokku.app-name=$APP" + containers="$("$DOCKER_BIN" container ls --filter "$container_type_filter" --filter "$app_filter" --format '{{.ID}} {{.CreatedAt}}')" + else + containers="$("$DOCKER_BIN" container ls --filter "$container_type_filter" --format '{{.ID}} {{.CreatedAt}}')" + fi + + if [[ -z "$containers" ]]; then + return + fi + + # iterate over all containers, ignoring the timezone in the last two columns + echo "$containers" | awk '{NF-=2} 1' | while read -r container_id container_date container_time; do + # get the cutoff time in seconds from the container's `com.dokku.active-deadline-seconds` label + active_deadline_seconds="$("$DOCKER_BIN" container inspect "$container_id" --format '{{ index .Config.Labels "com.dokku.active-deadline-seconds" }}')" + if [[ -z "$active_deadline_seconds" ]]; then + continue + fi + + # convert the active deadline seconds to a unix timestamp + cutoff_time="$(date --date="$active_deadline_seconds seconds ago" "+%s")" + + # convert the container start time to unix timestamp + start_time="$(date --date="$container_date $container_time" "+%s")" + + # if the container start time is before the cutoff time, stop the container + if [[ "$start_time" -lt "$cutoff_time" ]]; then + dokku_log_verbose_quiet "Retiring container ${container_id}" + fn-scheduler-docker-local-run-retire-container "$container_id" + fi + done +} + +trigger-scheduler-docker-local-scheduler-run-retire() { + declare desc="retires all run containers for an app that have exceeded their active deadline" + declare trigger="scheduler-run-retire" + declare DOKKU_SCHEDULER="$1" APP="$2" + + if [[ -z "$DOKKU_SCHEDULER" ]]; then + dokku_log_info1_quiet "Retiring all run containers" + fn-scheduler-docker-local-run-retire + return "$?" + fi + + if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then + return + fi + + dokku_log_info1_quiet "Retiring run containers for app ${APP}" + fn-scheduler-docker-local-run-retire "$APP" +} + +trigger-scheduler-docker-local-scheduler-run-retire "$@" diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index dcc6060d8..4f57700ba 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -349,24 +349,25 @@ type ClusterIssuer struct { } type Job struct { - AppName string - Command []string - DeploymentID int64 - Entrypoint string - Env map[string]string - ID string - Image string - ImagePullSecrets string - ImageSourceType string - Interactive bool - Labels map[string]string - Namespace string - ProcessType string - Schedule string - SecurityContext SecurityContext - Suffix string - RemoveContainer bool - WorkingDir string + AppName string + Command []string + DeploymentID int64 + Entrypoint string + Env map[string]string + ID string + Image string + ImagePullSecrets string + ImageSourceType string + Interactive bool + Labels map[string]string + Namespace string + ProcessType string + Schedule string + SecurityContext SecurityContext + Suffix string + RemoveContainer bool + WorkingDir string + ActiveDeadlineSeconds int64 } // SecurityContext contains the security context for a process @@ -479,7 +480,8 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { Annotations: annotations, }, Spec: batchv1.JobSpec{ - BackoffLimit: ptr.To(int32(0)), + ActiveDeadlineSeconds: ptr.To(input.ActiveDeadlineSeconds), + BackoffLimit: ptr.To(int32(0)), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index 102b3ce1e..4771674eb 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -50,6 +50,7 @@ spec: backoffLimit: 0 podReplacementPolicy: Failed ttlSecondsAfterFinished: 60 + activeDeadlineSeconds: 86400 template: metadata: annotations: diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index de2e9c2c9..1c12808ba 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -1329,23 +1329,32 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] return fmt.Errorf("Error getting security context: %w", err) } + activeDeadlineSeconds := int64(86400) // 24 hours + if os.Getenv("DOKKU_RUN_TTL_SECONDS") != "" { + activeDeadlineSeconds, err = strconv.ParseInt(os.Getenv("DOKKU_RUN_TTL_SECONDS"), 10, 64) + if err != nil { + return fmt.Errorf("Error parsing DOKKU_RUN_TTL_SECONDS value as int64: %w", err) + } + } + workingDir := common.GetWorkingDir(appName, image) job, err := templateKubernetesJob(Job{ - AppName: appName, - Command: []string{commandShell}, - DeploymentID: deploymentID, - Entrypoint: entrypoint, - Env: extraEnv, - Image: image, - ImagePullSecrets: imagePullSecrets, - ImageSourceType: imageSourceType, - Interactive: attachToPod || common.ToBool(os.Getenv("DOKKU_FORCE_TTY")), - Labels: labels, - Namespace: namespace, - ProcessType: processType, - RemoveContainer: rmContainer, - SecurityContext: securityContext, - WorkingDir: workingDir, + AppName: appName, + Command: []string{commandShell}, + DeploymentID: deploymentID, + Entrypoint: entrypoint, + Env: extraEnv, + Image: image, + ImagePullSecrets: imagePullSecrets, + ImageSourceType: imageSourceType, + Interactive: attachToPod || common.ToBool(os.Getenv("DOKKU_FORCE_TTY")), + Labels: labels, + Namespace: namespace, + ProcessType: processType, + RemoveContainer: rmContainer, + SecurityContext: securityContext, + WorkingDir: workingDir, + ActiveDeadlineSeconds: activeDeadlineSeconds, }) if err != nil { return fmt.Errorf("Error templating job: %w", err) diff --git a/tests/unit/run_1.bats b/tests/unit/run_1.bats index 913b6de2c..91fda6ce9 100644 --- a/tests/unit/run_1.bats +++ b/tests/unit/run_1.bats @@ -98,3 +98,45 @@ teardown() { echo "status: $status" assert_success } + +@test "(run) run:retire" { + run deploy_app + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku run:detached --ttl-seconds=1 $TEST_APP sleep 300" + echo "output: $output" + echo "status: $status" + assert_success + container_id="$output" + + # check the labels on the container to ensure the active deadline seconds is set + run /bin/bash -c "docker container inspect $container_id --format '{{ index .Config.Labels \"com.dokku.active-deadline-seconds\" }}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "1" + + run /bin/bash -c "dokku run:list --quiet $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "$container_id" + + run /bin/bash -c "sleep 2" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku run:retire" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku run:list --quiet $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_not_contains "$container_id" +} diff --git a/tests/unit/run_2.bats b/tests/unit/run_2.bats index d6f8e860a..bd461aa31 100644 --- a/tests/unit/run_2.bats +++ b/tests/unit/run_2.bats @@ -112,12 +112,11 @@ teardown() { echo "status: $status" assert_success - # includes headers run /bin/bash -c "dokku --quiet run:list $TEST_APP | wc -l" echo "output: $output" echo "status: $status" assert_success - assert_output "2" + assert_output "1" run /bin/bash -c "dokku run:list $TEST_APP --format json | jq '. | length'" echo "output: $output" @@ -136,12 +135,11 @@ teardown() { echo "status: $status" assert_success - # includes headers run /bin/bash -c "dokku --quiet run:list $TEST_APP | wc -l" echo "output: $output" echo "status: $status" assert_success - assert_output "3" + assert_output "2" run /bin/bash -c "dokku run:list $TEST_APP --format json | jq '. | length'" echo "output: $output"