Merge pull request #7991 from dokku/7145-cron-maintenance-id

Add ability to pause/restart cron jobs

docs/appendices/file-formats/app-json.md

@@ -21,6 +21,7 @@
 (list, optional) A list of cron resources. Keys are the names of the process types. The values are an object containing one or more of the following properties:
 
 - `command`: (string, required)
+- `maintenance`: (boolean, optional)
 - `schedule`: (string, required)
 
 ## Formation

docs/processes/scheduled-cron-tasks.md

@@ -6,8 +6,10 @@
 ```
 cron:list <app> [--format json|stdout]  # List scheduled cron tasks for an app
 cron:report [<app>] [<flag>]            # Display report about an app
+cron:resume <app> <cron_id>             # Resume a cron task
 cron:run <app> <cron_id> [--detach]     # Run a cron task on the fly
 cron:set [--global|<app>] <key> <value> # Set or clear a cron property for an app
+cron:suspend <app> <cron_id>            # Suspend a cron task
 ```
 
 ## Usage
@@ -34,6 +36,7 @@ The `app.json` file for a given app can define a special `cron` key that contain
 A cron task takes the following properties:
 
 - `command`: A command to be run within the built app image. Specified commands can also be `Procfile` entries.
+- `maintenance`: A boolean value that decides whether the cron task is in maintenance and therefore executable or not.
 - `schedule`: A [cron-compatible](https://en.wikipedia.org/wiki/Cron#Overview) scheduling definition upon which to run the command. Seconds are generally not supported.
 
 Zero or more cron tasks can be specified per app. Cron tasks are validated after the build artifact is created but before the app is deployed, and the cron schedule is updated during the post-deploy phase.
@@ -123,6 +126,26 @@ ID                            Schedule  Command
 5cruaotm4yzzpnjlsdunblj8qyjp  @daily    /bin/true
 ```
 
+#### Suspending and resuming a specific cron task
+
+Cron tasks can be suspended to temporarily prevent them from running, and later resumed to re-enable them. This is useful for maintenance or debugging purposes.
+
+To suspend a specific cron task, use the `cron:suspend` command with the app name and cron ID:
+
+```shell
+dokku cron:suspend node-js-app cGhwPT09cGhwIHRlc3QucGhwPT09QGRhaWx5
+```
+
+A suspended task will not execute according to its schedule. You can verify a task is suspended by checking the `Maintenance` column in the `cron:list` output, which will show `true (task)` for suspended tasks.
+
+To resume a suspended cron task, use the `cron:resume` command:
+
+```shell
+dokku cron:resume node-js-app cGhwPT09cGhwIHRlc3QucGhwPT09QGRhaWx5
+```
+
+Once resumed, the task will execute according to its schedule again. The cron ID can be retrieved from the `cron:list` output.
+
 #### Executing a cron task on the fly
 
 Cron tasks can be invoked via the `cron:run` command. This command takes an `app` argument and a `cron id` (retrievable from `cron:list` output).

plugins/app-json/appjson.go

@@ -55,6 +55,9 @@ type CronTask struct {
 	// Command is the command to execute
 	Command string `json:"command"`
 
+	// Maintenance is whether or not the cron task is in maintenance mode
+	Maintenance bool `json:"maintenance"`
+
 	// Schedule is the cron schedule to execute the command on
 	Schedule string `json:"schedule"`
 }

plugins/common/properties.go

@@ -12,27 +12,27 @@ import (
 )
 
 // CommandPropertySet is a generic function that will set a property for a given plugin/app combination
-func CommandPropertySet(pluginName, appName, property, value string, properties map[string]string, globalProperties map[string]bool) {
+func CommandPropertySet(pluginName, appName, property, value string, validProperties map[string]string, validGlobalProperties map[string]bool) {
 	if appName != "--global" {
 		if err := VerifyAppName(appName); err != nil {
 			LogFailWithError(err)
 		}
 	}
-	if appName == "--global" && !globalProperties[property] {
+	if appName == "--global" && !validGlobalProperties[property] {
 		LogFail("Property cannot be specified globally")
 	}
 	if property == "" {
 		LogFail("No property specified")
 	}
 
-	for k := range globalProperties {
-		if _, ok := properties[k]; !ok {
-			properties[k] = ""
+	for k := range validGlobalProperties {
+		if _, ok := validProperties[k]; !ok {
+			validProperties[k] = ""
 		}
 	}
 
-	if _, ok := properties[property]; !ok {
-		properties := reflect.ValueOf(properties).MapKeys()
+	if _, ok := validProperties[property]; !ok {
+		properties := reflect.ValueOf(validProperties).MapKeys()
 		validPropertyList := make([]string, len(properties))
 		for i := 0; i < len(properties); i++ {
 			validPropertyList[i] = properties[i].String()

plugins/cron/Makefile

@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/list subcommands/report subcommands/run subcommands/set
+SUBCOMMANDS = subcommands/list subcommands/report subcommands/resume subcommands/run subcommands/set subcommands/suspend
 TRIGGERS = triggers/app-json-is-valid triggers/cron-get-property triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = cron

plugins/cron/cron.go

@@ -2,6 +2,7 @@ package cron
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	appjson "github.com/dokku/dokku/plugins/app-json"
@@ -27,6 +28,8 @@ var (
 	}
 )
 
+const MaintenancePropertyPrefix = "maintenance."
+
 // CronTask is a struct that represents a cron task
 type CronTask struct {
 	// ID is a unique identifier for the cron task
@@ -50,6 +53,12 @@ type CronTask struct {
 	// LogFile is the log file to write to
 	LogFile string `json:"-"`
 
+	// AppInMaintenance is whether the app's cron is in maintenance mode
+	AppInMaintenance bool `json:"app-in-maintenance"`
+
+	// Maintenance is whether the cron task is in maintenance mode
+	TaskInMaintenance bool `json:"task-in-maintenance"`
+
 	// Maintenance is whether the cron task is in maintenance mode
 	Maintenance bool `json:"maintenance"`
 }
@@ -77,7 +86,7 @@ type FetchCronTasksInput struct {
 func FetchCronTasks(input FetchCronTasksInput) ([]CronTask, error) {
 	appName := input.AppName
 	tasks := []CronTask{}
-	isMaintenance := reportComputedMaintenance(appName) == "true"
+	isAppCronInMaintenance := reportComputedMaintenance(appName) == "true"
 
 	if input.AppJSON == nil && input.AppName == "" {
 		return tasks, fmt.Errorf("Missing app name or app.json")
@@ -96,6 +105,11 @@ func FetchCronTasks(input FetchCronTasksInput) ([]CronTask, error) {
 		return tasks, nil
 	}
 
+	properties, err := common.PropertyGetAllByPrefix("cron", appName, MaintenancePropertyPrefix)
+	if err != nil {
+		return tasks, fmt.Errorf("Error getting maintenance properties: %w", err)
+	}
+
 	for i, c := range input.AppJSON.Cron {
 		if c.Command == "" {
 			if input.WarnToFailure {
@@ -121,12 +135,28 @@ func FetchCronTasks(input FetchCronTasksInput) ([]CronTask, error) {
 			return tasks, fmt.Errorf("Invalid cron schedule for app %s (schedule %s): %s", appName, c.Schedule, err.Error())
 		}
 
+		cronID := GenerateCommandID(appName, c)
+		maintenance := c.Maintenance
+		if value, ok := properties[MaintenancePropertyPrefix+cronID]; ok {
+			boolValue, err := strconv.ParseBool(value)
+			if err != nil {
+				return tasks, fmt.Errorf("Invalid maintenance property for app %s (schedule %s): %s", appName, c.Schedule, err.Error())
+			}
+
+			// only override the maintenance value if the property is set to true
+			if boolValue {
+				maintenance = boolValue
+			}
+		}
+
 		tasks = append(tasks, CronTask{
-			App:         appName,
-			Command:     c.Command,
-			Schedule:    c.Schedule,
-			ID:          GenerateCommandID(appName, c),
-			Maintenance: isMaintenance,
+			App:               appName,
+			Command:           c.Command,
+			Schedule:          c.Schedule,
+			ID:                cronID,
+			Maintenance:       isAppCronInMaintenance || maintenance,
+			AppInMaintenance:  isAppCronInMaintenance,
+			TaskInMaintenance: maintenance,
 		})
 	}
 
@@ -155,12 +185,14 @@ func FetchGlobalCronTasks() ([]CronTask, error) {
 
 		id := base36.EncodeToStringLc([]byte(strings.Join(parts, ";;;")))
 		task := CronTask{
-			ID:          id,
-			Schedule:    parts[0],
-			Command:     parts[1],
-			AltCommand:  parts[1],
-			Maintenance: false,
-			Global:      true,
+			ID:                id,
+			Schedule:          parts[0],
+			Command:           parts[1],
+			AltCommand:        parts[1],
+			Global:            true,
+			Maintenance:       false,
+			TaskInMaintenance: false,
+			AppInMaintenance:  false,
 		}
 		if len(parts) == 3 {
 			task.LogFile = parts[2]

plugins/cron/report.go

@@ -1,7 +1,9 @@
 package cron
 
 import (
+	"fmt"
 	"strconv"
+	"strings"
 
 	"github.com/dokku/dokku/plugins/common"
 )
@@ -21,6 +23,11 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		"--cron-maintenance":          reportMaintenance,
 	}
 
+	extraFlags := addCronMaintenanceFlags(appName, infoFlag)
+	for flag, fn := range extraFlags {
+		flags[flag] = fn
+	}
+
 	flagKeys := []string{}
 	for flagKey := range flags {
 		flagKeys = append(flagKeys, flagKey)
@@ -32,6 +39,24 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 	return common.ReportSingleApp("cron", appName, infoFlag, infoFlags, flagKeys, format, trimPrefix, uppercaseFirstCharacter)
 }
 
+func addCronMaintenanceFlags(appName string, infoFlag string) map[string]common.ReportFunc {
+	flags := map[string]common.ReportFunc{}
+
+	properties, err := common.PropertyGetAllByPrefix("cron", appName, MaintenancePropertyPrefix)
+	if err != nil {
+		return flags
+	}
+
+	for property, value := range properties {
+		key := strings.Replace(property, MaintenancePropertyPrefix, "", 1)
+		flags[fmt.Sprintf("--cron-maintenance-%s", key)] = func(appName string) string {
+			return value
+		}
+	}
+
+	return flags
+}
+
 func reportMailfrom(_ string) string {
 	return common.PropertyGet("cron", "--global", "mailfrom")
 }

plugins/cron/src/commands/commands.go

@@ -20,8 +20,10 @@ Additional commands:`
 	helpContent = `
     cron:list <app> [--format json|stdout], List scheduled cron tasks for an app
     cron:report [<app>] [<flag>], Display report about an app
+    cron:resume <app> <cron_id>, Resume a cron task
     cron:run <app> <cron_id> [--detach], Run a cron task on the fly
-    cron:set [--global|<app>] <key> <value>, Set or clear a cron property for an app`
+    cron:set [--global|<app>] <key> <value>, Set or clear a cron property for an app
+    cron:suspend <app> <cron_id>, Suspend a cron task`
 )
 
 func main() {

plugins/cron/src/subcommands/subcommands.go

@@ -37,6 +37,12 @@ func main() {
 			appName := args.Arg(0)
 			err = cron.CommandReport(appName, *format, infoFlag)
 		}
+	case "resume":
+		args := flag.NewFlagSet("cron:resume", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		appName := args.Arg(0)
+		cronID := args.Arg(1)
+		err = cron.CommandResume(appName, cronID)
 	case "run":
 		args := flag.NewFlagSet("cron:run", flag.ExitOnError)
 		detached := args.Bool("detach", false, "--detach: run the container in a detached mode")
@@ -57,6 +63,12 @@ func main() {
 			value = args.Arg(1)
 		}
 		err = cron.CommandSet(appName, property, value)
+	case "suspend":
+		args := flag.NewFlagSet("cron:suspend", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		appName := args.Arg(0)
+		cronID := args.Arg(1)
+		err = cron.CommandSuspend(appName, cronID)
 	default:
 		err = fmt.Errorf("Invalid plugin subcommand call: %s", subcommand)
 	}

plugins/cron/subcommands.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/dokku/dokku/plugins/common"
 
@@ -43,7 +44,15 @@ func CommandList(appName string, format string) error {
 	if format == "stdout" {
 		output := []string{"ID | Schedule | Maintenance | Command"}
 		for _, task := range tasks {
-			output = append(output, fmt.Sprintf("%s | %s | %t | %s", task.ID, task.Schedule, task.Maintenance, task.Command))
+			maintenance := "false"
+			if task.Maintenance {
+				if task.TaskInMaintenance {
+					maintenance = "true (task)"
+				} else if task.AppInMaintenance {
+					maintenance = "true (app)"
+				}
+			}
+			output = append(output, fmt.Sprintf("%s | %s | %s | %s", task.ID, task.Schedule, maintenance, task.Command))
 		}
 
 		result := columnize.SimpleFormat(output)
@@ -82,6 +91,11 @@ func CommandReport(appName string, format string, infoFlag string) error {
 	return ReportSingleApp(appName, format, infoFlag)
 }
 
+// CommandResume resumes a cron task
+func CommandResume(appName string, cronID string) error {
+	return CommandSet(appName, fmt.Sprintf("%s%s", MaintenancePropertyPrefix, cronID), "")
+}
+
 // CommandRun executes a cron task on the fly
 func CommandRun(appName string, cronID string, detached bool) error {
 	if err := common.VerifyAppName(appName); err != nil {
@@ -138,7 +152,37 @@ func CommandSet(appName string, property string, value string) error {
 		return err
 	}
 
-	common.CommandPropertySet("cron", appName, property, value, DefaultProperties, GlobalProperties)
+	validProperties := DefaultProperties
+	globalProperties := GlobalProperties
+	if strings.HasPrefix(property, MaintenancePropertyPrefix) {
+		if appName == "--global" {
+			return fmt.Errorf("Task maintenance properties cannot be set globally")
+		}
+
+		cronTaskID := strings.TrimPrefix(property, MaintenancePropertyPrefix)
+		if cronTaskID == "" {
+			return fmt.Errorf("Invalid task maintenance property, missing ID")
+		}
+
+		tasks, err := FetchCronTasks(FetchCronTasksInput{AppName: appName})
+		if err != nil {
+			return err
+		}
+
+		for _, task := range tasks {
+			if task.ID == cronTaskID {
+				validProperties[property] = ""
+				globalProperties[property] = false
+				break
+			}
+		}
+
+		if _, ok := validProperties[property]; !ok {
+			return fmt.Errorf("Invalid task maintenance property, no matching task ID found: %s", property)
+		}
+	}
+
+	common.CommandPropertySet("cron", appName, property, value, validProperties, globalProperties)
 	scheduler := common.GetAppScheduler(appName)
 	_, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
 		Trigger:     "scheduler-cron-write",
@@ -147,3 +191,8 @@ func CommandSet(appName string, property string, value string) error {
 	})
 	return err
 }
+
+// CommandSuspend suspends a cron task
+func CommandSuspend(appName string, cronID string) error {
+	return CommandSet(appName, fmt.Sprintf("%s%s", MaintenancePropertyPrefix, cronID), "true")
+}

plugins/scheduler-docker-local/functions.go

@@ -107,16 +107,17 @@ func generateCronTasks() ([]cron.CronTask, error) {
 	}
 
 	for result := range results {
-		c := result
-		if len(c) > 0 && !c[0].Maintenance {
-			tasks = append(tasks, c...)
+		for _, task := range result {
+			if !task.Maintenance {
+				tasks = append(tasks, task)
+			}
 		}
 	}
 
 	return tasks, nil
 }
 
-func writeCronTasks(scheduler string) error {
+func writeCronTab(scheduler string) error {
 	// allow empty scheduler, which means all apps (used by letsencrypt)
 	if scheduler != "docker-local" && scheduler != "" {
 		return nil
@@ -154,7 +155,7 @@ func writeCronTasks(scheduler string) error {
 		return err
 	}
 
-	tmpFile, err := os.CreateTemp(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "WriteCronTasks"))
+	tmpFile, err := os.CreateTemp(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "WriteCronTab"))
 	if err != nil {
 		return fmt.Errorf("Cannot create temporary schedule file: %v", err)
 	}

plugins/scheduler-docker-local/triggers.go

@@ -2,5 +2,5 @@ package schedulerdockerlocal
 
 // TriggerSchedulerCronWrite force updates the cron file for all apps
 func TriggerSchedulerCronWrite(scheduler string) error {
-	return writeCronTasks(scheduler)
+	return writeCronTab(scheduler)
 }

plugins/scheduler-k3s/Makefile

@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
-TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
+TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 

plugins/scheduler-k3s/k8s.go

@@ -805,6 +805,35 @@ func (k KubernetesClient) ListTriggerAuthentications(ctx context.Context, input
 	return triggerAuthentications, nil
 }
 
+// ResumeCronJobsInput contains all the information needed to resume a Kubernetes cron job
+type ResumeCronJobsInput struct {
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// ResumeCronJobs resumes a Kubernetes cron job
+func (k KubernetesClient) ResumeCronJobs(ctx context.Context, input ResumeCronJobsInput) error {
+	cronJobs, err := k.Client.BatchV1().CronJobs(input.Namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: input.LabelSelector,
+	})
+	if err != nil {
+		return err
+	}
+
+	for _, cronJob := range cronJobs.Items {
+		cronJob.Spec.Suspend = ptr.To(false)
+		_, err := k.Client.BatchV1().CronJobs(input.Namespace).Update(ctx, &cronJob, metav1.UpdateOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // ScaleDeploymentInput contains all the information needed to scale a Kubernetes deployment
 type ScaleDeploymentInput struct {
 	// Name is the Kubernetes deployment name

plugins/scheduler-k3s/src/triggers/triggers.go

@@ -123,6 +123,10 @@ func main() {
 		scheduler := flag.Arg(0)
 		appName := flag.Arg(1)
 		err = scheduler_k3s.TriggerSchedulerStop(scheduler, appName)
+	case "scheduler-cron-write":
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		err = scheduler_k3s.TriggerSchedulerCronWrite(scheduler, appName)
 	case "scheduler-post-delete":
 		scheduler := flag.Arg(0)
 		appName := flag.Arg(1)

plugins/scheduler-k3s/template.go

@@ -302,6 +302,7 @@ type ProcessCron struct {
 	ID       string `yaml:"id"`
 	Schedule string `yaml:"schedule"`
 	Suffix   string `yaml:"suffix"`
+	Suspend  bool   `yaml:"suspend"`
 }
 
 type ProcessPortMap struct {

plugins/scheduler-k3s/templates/chart/cron-job.yaml

@@ -135,6 +135,6 @@ spec:
   schedule: {{ $config.cron.schedule }}
   startingDeadlineSeconds: 60
   successfulJobsHistoryLimit: 10
-  suspend: false
+  suspend: {{ $config.cron.suspend }}
   timeZone: Etc/UTC
 {{- end }}

plugins/scheduler-k3s/triggers.go

@@ -163,6 +163,64 @@ func TriggerSchedulerAppStatus(scheduler string, appName string) error {
 	return nil
 }
 
+// TriggerSchedulerCronWrite writes out cron tasks for a given application
+func TriggerSchedulerCronWrite(scheduler string, appName string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+
+	cronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
+	if err != nil {
+		return fmt.Errorf("Error fetching cron tasks: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", appName))
+		cancel()
+	}()
+
+	namespace := getComputedNamespace(appName)
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	for _, cronTask := range cronTasks {
+		labelSelector := []string{
+			fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
+			fmt.Sprintf("dokku.com/cron-id=%s", cronTask.ID),
+		}
+
+		if cronTask.Maintenance {
+			err = clientset.SuspendCronJobs(ctx, SuspendCronJobsInput{
+				Namespace:     namespace,
+				LabelSelector: strings.Join(labelSelector, ","),
+			})
+			if err != nil {
+				return fmt.Errorf("Error suspending cron jobs: %w", err)
+			}
+		} else {
+			err = clientset.ResumeCronJobs(ctx, ResumeCronJobsInput{
+				Namespace:     namespace,
+				LabelSelector: strings.Join(labelSelector, ","),
+			})
+			if err != nil {
+				return fmt.Errorf("Error resuming cron jobs: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
 // TriggerSchedulerDeploy deploys an image tag for a given application
 func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) error {
 	if scheduler != "k3s" {
@@ -312,17 +370,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 
 	workingDir := common.GetWorkingDir(appName, image)
 
-	allCronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
+	cronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
 	if err != nil {
 		return fmt.Errorf("Error fetching cron tasks: %w", err)
 	}
-	// remove maintenance cron tasks
-	cronTasks := []cron.CronTask{}
-	for _, cronTask := range allCronTasks {
-		if !cronTask.Maintenance {
-			cronTasks = append(cronTasks, cronTask)
-		}
-	}
 
 	domains := []string{}
 	if _, ok := processes["web"]; ok {
@@ -655,6 +706,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 				ID:       cronTask.ID,
 				Schedule: cronTask.Schedule,
 				Suffix:   suffix,
+				Suspend:  cronTask.Maintenance,
 			},
 			Labels:      labels,
 			ProcessType: ProcessType_Cron,

tests/unit/cron.bats

@@ -129,7 +129,7 @@ teardown() {
   echo "output: $output"
   echo "status: $status"
   assert_success
-  assert_output '[{"id":"5cruaotm4yzzpnjlsdunblj8qyjp","command":"/bin/true","global":true,"schedule":"@daily","maintenance":false}]'
+  assert_output '[{"id":"5cruaotm4yzzpnjlsdunblj8qyjp","command":"/bin/true","global":true,"schedule":"@daily","app-in-maintenance":false,"task-in-maintenance":false,"maintenance":false}]'
 
   run /bin/bash -c "cat /var/spool/cron/crontabs/dokku"
   echo "output: $output"
@@ -235,6 +235,74 @@ teardown() {
   assert_output "['task.py', 'schedule', 'now']"
 }
 
+@test "(cron) cron:suspend cron:resume" {
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP template_cron_file_valid_multiple
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /var/spool/cron/crontabs/dokku"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "python3 task.py first"
+  assert_output_contains "python3 task.py second"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  first_command="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].command')"
+  run /bin/bash -c "echo $first_command"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku cron:report $TEST_APP --cron-maintenance-$cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "dokku cron:suspend $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku cron:report $TEST_APP --cron-maintenance-$cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "true"
+
+  run /bin/bash -c "cat /var/spool/cron/crontabs/dokku"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "python3 task.py first" 0
+  assert_output_contains "python3 task.py second"
+
+  run /bin/bash -c "dokku cron:resume $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku cron:report $TEST_APP --cron-maintenance-$cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "cat /var/spool/cron/crontabs/dokku"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "python3 task.py first"
+  assert_output_contains "python3 task.py second"
+}
+
 template_cron_file_invalid() {
   local APP="$1"
   local APP_REPO_DIR="$2"