Merge pull request #7579 from dokku/6872-stop-all

Stop all app containers when calling ps:stop

docs/deployment/schedulers/scheduler-management.md

@@ -98,7 +98,7 @@ At this time, the following dokku commands are used to implement a complete sche
 - `apps:clone`: handles app cloning
     - triggers: post-app-clone-setup
 - `cron`: generates cron tasks for the app
-    - triggers: cron-write
+    - triggers: scheduler-cron-write
 - `deploy`: deploys app proceses and checks the status of a deploy
     - triggers: scheduler-app-status, scheduler-deploy, scheduler-is-deployed, scheduler-logs-failed
 - `enter`: enters a running container
@@ -112,7 +112,9 @@ At this time, the following dokku commands are used to implement a complete sche
 - `run:logs`: fetches running container logs
     - triggers: scheduler-run-logs
 - `ps:stop`: stops app processes
-    - triggers: scheduler-stop
+    - triggers: scheduler-stop, scheduler-cron-write
+- `ps:start`: starts app processes
+    - triggers: scheduler-deploy, scheduler-cron-write
 - `ps:inspect`: outputs inspect output for processes in an app
     - triggers: scheduler-inspect
 

docs/development/plugin-triggers.md

@@ -119,6 +119,21 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
 # TODO
 ```
 
+### `app-json-is-valid`
+
+- Description: Checks to see if the provided app.json file is valid
+- Invoked by: Appjson extraction during deployment
+- Arguments: `$APP $APP_JSON_PATH`
+- Example:
+
+```shell
+#!/usr/bin/env bash
+
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+
+# TODO
+```
+
 ### `app-maybe-create`
 
 - Description: Creates an app (gated by whether this is globally enabled or not)
@@ -583,22 +598,6 @@ DOKKU_SCHEDULER="$1"
 # TODO
 ```
 
-### `cron-write`
-
-- Description: Force triggers writing out cron entries
-- Invoked by:
-- Arguments:
-- Example:
-
-```shell
-#!/usr/bin/env bash
-
-set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
-source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
-
-# TODO
-```
-
 ### `dependencies`
 
 - Description: Used to install system-level dependencies.
@@ -2445,6 +2444,26 @@ DOKKU_SCHEDULER="$1"; APP="$2";
 # TODO
 ```
 
+### `scheduler-cron-write`
+
+> [!WARNING]
+> The scheduler plugin trigger apis are under development and may change
+> between minor releases until the 1.0 release.
+
+- Description: Force triggers writing out cron entries. Arguments are optional.
+- Invoked by: `ps:start`, `ps:stop`, `cron:set`
+- Arguments: `$DOKKU_SCHEDULER $APP`
+- Example:
+
+```shell
+#!/usr/bin/env bash
+
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+
+# TODO
+```
+
 ### `scheduler-deploy`
 
 > [!WARNING]

plugins/app-json/appjson.go

@@ -200,21 +200,9 @@ func GetAppJSON(appName string) (AppJSON, error) {
 		return AppJSON{}, nil
 	}
 
-	b, err := os.ReadFile(getProcessSpecificAppJSONPath(appName))
-	if err != nil {
-		return AppJSON{}, fmt.Errorf("Cannot read app.json file: %v", err)
-	}
+	appJSONPath := getProcessSpecificAppJSONPath(appName)
 
-	if strings.TrimSpace(string(b)) == "" {
-		return AppJSON{}, nil
-	}
-
-	var appJSON AppJSON
-	if err = json.Unmarshal(b, &appJSON); err != nil {
-		return AppJSON{}, fmt.Errorf("Cannot parse app.json: %v", err)
-	}
-
-	return appJSON, nil
+	return ReadAppJSON(appJSONPath)
 }
 
 func GetAutoscalingConfig(appName string, processType string, replicas int) (FormationAutoscaling, bool, error) {
@@ -256,3 +244,22 @@ func GetAutoscalingConfig(appName string, processType string, replicas int) (For
 
 	return autoscaling, true, nil
 }
+
+// ReadAppJSON reads an app.json file from a given path
+func ReadAppJSON(path string) (AppJSON, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return AppJSON{}, fmt.Errorf("Cannot read app.json file: %v", err)
+	}
+
+	if strings.TrimSpace(string(b)) == "" {
+		return AppJSON{}, nil
+	}
+
+	var appJSON AppJSON
+	if err = json.Unmarshal(b, &appJSON); err != nil {
+		return AppJSON{}, fmt.Errorf("Cannot parse app.json: %v", err)
+	}
+
+	return appJSON, nil
+}

plugins/app-json/triggers.go

@@ -1,6 +1,7 @@
 package appjson
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -140,6 +141,22 @@ func TriggerCorePostExtract(appName string, sourceWorkDir string) error {
 		}
 	}
 
+	if common.FileExists(processSpecificAppJSON) {
+		result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+			Trigger:      "app-json-is-valid",
+			Args:         []string{appName, processSpecificAppJSON},
+			StreamStdout: true,
+			StreamStderr: true,
+		})
+		if err != nil {
+			if result.StderrContents() != "" {
+				return errors.New(result.StderrContents())
+			}
+
+			return err
+		}
+	}
+
 	// TODO: add validation to app.json file by ensuring it can be deserialized
 	return nil
 }

plugins/cron/Makefile

@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/list subcommands/report subcommands/run subcommands/set
-TRIGGERS = triggers/cron-get-property triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/scheduler-stop
+TRIGGERS = triggers/app-json-is-valid triggers/cron-get-property triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = cron
 

plugins/cron/cron.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	appjson "github.com/dokku/dokku/plugins/app-json"
+	"github.com/dokku/dokku/plugins/common"
 
 	"github.com/multiformats/go-base36"
 	cronparser "github.com/robfig/cron/v3"
@@ -61,25 +62,55 @@ func (t TemplateCommand) CronCommand() string {
 	return fmt.Sprintf("dokku run --cron-id %s %s %s", t.ID, t.App, t.Command)
 }
 
+// FetchCronEntriesInput is the input for the FetchCronEntries function
+type FetchCronEntriesInput struct {
+	AppName       string
+	AppJSON       *appjson.AppJSON
+	WarnToFailure bool
+}
+
 // FetchCronEntries returns a list of cron commands for a given app
-func FetchCronEntries(appName string) ([]TemplateCommand, error) {
+func FetchCronEntries(input FetchCronEntriesInput) ([]TemplateCommand, error) {
+	appName := input.AppName
 	commands := []TemplateCommand{}
 	isMaintenance := reportComputedMaintenance(appName) == "true"
 
-	appJSON, err := appjson.GetAppJSON(appName)
-	if err != nil {
-		return commands, fmt.Errorf("Unable to fetch app.json for app %s: %s", appName, err.Error())
+	if input.AppJSON == nil {
+		appJSON, err := appjson.GetAppJSON(appName)
+		if err != nil {
+			return commands, fmt.Errorf("Unable to fetch app.json for app %s: %s", appName, err.Error())
+		}
+
+		input.AppJSON = &appJSON
 	}
 
-	if appJSON.Cron == nil {
+	if input.AppJSON.Cron == nil {
 		return commands, nil
 	}
 
-	for _, c := range appJSON.Cron {
+	for i, c := range input.AppJSON.Cron {
+		if c.Command == "" {
+			if input.WarnToFailure {
+				return commands, fmt.Errorf("Missing cron command for app %s (index %d)", appName, i)
+			}
+
+			common.LogWarn(fmt.Sprintf("Missing cron command for app %s (index %d)", appName, i))
+			continue
+		}
+
+		if c.Schedule == "" {
+			if input.WarnToFailure {
+				return commands, fmt.Errorf("Missing cron schedule for app %s (index %d)", appName, i)
+			}
+
+			common.LogWarn(fmt.Sprintf("Missing cron schedule for app %s (index %d)", appName, i))
+			continue
+		}
+
 		parser := cronparser.NewParser(cronparser.Minute | cronparser.Hour | cronparser.Dom | cronparser.Month | cronparser.Dow | cronparser.Descriptor)
 		_, err := parser.Parse(c.Schedule)
 		if err != nil {
-			return commands, fmt.Errorf("Invalid cron schedule %s: %s", c.Schedule, err.Error())
+			return commands, fmt.Errorf("Invalid cron schedule for app %s (schedule %s): %s", appName, c.Schedule, err.Error())
 		}
 
 		commands = append(commands, TemplateCommand{

plugins/cron/report.go

@@ -41,7 +41,7 @@ func reportMailto(_ string) string {
 }
 
 func reportTasks(appName string) string {
-	c, _ := FetchCronEntries(appName)
+	c, _ := FetchCronEntries(FetchCronEntriesInput{AppName: appName})
 	return strconv.Itoa(len(c))
 }
 

plugins/cron/src/triggers/triggers.go

@@ -19,6 +19,10 @@ func main() {
 
 	var err error
 	switch trigger {
+	case "app-json-is-valid":
+		appName := flag.Arg(0)
+		appJSONPath := flag.Arg(1)
+		err = cron.TriggerAppJSONIsValid(appName, appJSONPath)
 	case "cron-get-property":
 		appName := flag.Arg(0)
 		property := flag.Arg(1)

plugins/cron/subcommands.go

@@ -26,7 +26,7 @@ func CommandList(appName string, format string) error {
 		return fmt.Errorf("Invalid format specified, supported formats: json, stdout")
 	}
 
-	entries, err := FetchCronEntries(appName)
+	entries, err := FetchCronEntries(FetchCronEntriesInput{AppName: appName})
 	if err != nil {
 		return err
 	}
@@ -79,7 +79,7 @@ func CommandRun(appName string, cronID string, detached bool) error {
 		return err
 	}
 
-	entries, err := FetchCronEntries(appName)
+	entries, err := FetchCronEntries(FetchCronEntriesInput{AppName: appName})
 	if err != nil {
 		return err
 	}
@@ -130,8 +130,10 @@ func CommandSet(appName string, property string, value string) error {
 	}
 
 	common.CommandPropertySet("cron", appName, property, value, DefaultProperties, GlobalProperties)
+	scheduler := common.GetAppScheduler(appName)
 	_, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
-		Trigger:     "cron-write",
+		Trigger:     "scheduler-cron-write",
+		Args:        []string{scheduler, appName},
 		StreamStdio: true,
 	})
 	return err

plugins/cron/triggers.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 
+	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
 )
 
@@ -23,6 +24,29 @@ func TriggerCronGetProperty(appName string, key string) error {
 	return nil
 }
 
+// TriggerAppJSONIsValid validates the cron entries for a given app
+func TriggerAppJSONIsValid(appName string, appJSONPath string) error {
+	if !common.FileExists(appJSONPath) {
+		return nil
+	}
+
+	appJSON, err := appjson.ReadAppJSON(appJSONPath)
+	if err != nil {
+		return err
+	}
+
+	_, err = FetchCronEntries(FetchCronEntriesInput{
+		AppName:       appName,
+		AppJSON:       &appJSON,
+		WarnToFailure: true,
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // TriggerInstall runs the install step for the cron plugin
 func TriggerInstall() error {
 	if err := common.PropertySetup("cron"); err != nil {
@@ -57,7 +81,21 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
 
 // TriggerPostDelete destroys the cron property for a given app container
 func TriggerPostDelete(appName string) error {
-	return common.PropertyDestroy("cron", appName)
+	scheduler := common.GetAppScheduler(appName)
+	_, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Trigger:     "scheduler-cron-write",
+		Args:        []string{scheduler, appName},
+		StreamStdio: true,
+	})
+	if err != nil {
+		return err
+	}
+
+	if err := common.PropertyDestroy("cron", appName); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // TriggerSchedulerStop stops the scheduler for a given app container

plugins/scheduler-docker-local/Makefile

@@ -1,4 +1,4 @@
-TRIGGERS = triggers/cron-write
+TRIGGERS = triggers/scheduler-cron-write
 BUILD = triggers
 PLUGIN_NAME = scheduler-docker-local
 

plugins/scheduler-docker-local/functions.go

@@ -52,10 +52,11 @@ func generateCronEntries() ([]cron.TemplateCommand, error) {
 				return nil
 			}
 
-			c, err := cron.FetchCronEntries(appName)
+			c, err := cron.FetchCronEntries(cron.FetchCronEntriesInput{AppName: appName})
 			if err != nil {
 				results <- []cron.TemplateCommand{}
-				return err
+				common.LogWarn(err.Error())
+				return nil
 			}
 
 			results <- c
@@ -115,7 +116,12 @@ func generateCronEntries() ([]cron.TemplateCommand, error) {
 	return commands, nil
 }
 
-func writeCronEntries() error {
+func writeCronEntries(scheduler string) error {
+	// allow empty scheduler, which means all apps (used by letsencrypt)
+	if scheduler != "docker-local" && scheduler != "" {
+		return nil
+	}
+
 	commands, err := generateCronEntries()
 	if err != nil {
 		return err

plugins/scheduler-docker-local/post-delete

@@ -14,8 +14,6 @@ trigger-scheduler-docker-local-post-delete() {
 
   # remove all application containers
   "$DOCKER_BIN" container ls --filter "label=com.dokku.app-name=${APP}" -q | xargs -n1 -I {} "$DOCKER_BIN" container rm --force {} &>/dev/null || true
-
-  plugn trigger cron-write
 }
 
 trigger-scheduler-docker-local-post-delete "$@"

plugins/scheduler-docker-local/post-deploy

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -eo pipefail
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 [[ $DOKKU_TRACE ]] && set -x
 
 trigger-scheduler-docker-local-post-deploy() {
@@ -7,7 +8,8 @@ trigger-scheduler-docker-local-post-deploy() {
   declare trigger="post-deploy"
   declare APP="$1"
 
-  plugn trigger cron-write
+  local DOKKU_SCHEDULER="$(get_app_scheduler "$APP")"
+  plugn trigger scheduler-cron-write "$DOKKU_SCHEDULER" "$APP"
 }
 
 trigger-scheduler-docker-local-post-deploy "$@"

plugins/scheduler-docker-local/scheduler-stop

@@ -4,6 +4,12 @@ set -eo pipefail
 source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 source "$PLUGIN_AVAILABLE_PATH/config/functions"
 
+fn-stop-container() {
+  declare APP="$1" CID="$2"
+  "$DOCKER_BIN" container update --restart=no "$CID" &>/dev/null || true
+  "$DOCKER_BIN" container stop $DOCKER_STOP_TIME_ARG "$CID" &>/dev/null || true
+}
+
 trigger-scheduler-docker-local-scheduler-stop() {
   declare desc="scheduler-docker-local scheduler-stop plugin trigger"
   declare trigger="scheduler-stop"
@@ -19,14 +25,22 @@ trigger-scheduler-docker-local-scheduler-stop() {
   [[ -n "$DOKKU_DOCKER_STOP_TIMEOUT" ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}"
 
   if [[ -n "$DOKKU_APP_RUNNING_CONTAINER_IDS" ]]; then
-    # Disable the container restart policy
-    "$DOCKER_BIN" container update --restart=no $DOKKU_APP_RUNNING_CONTAINER_IDS &>/dev/null || true
+    for CID in $DOKKU_APP_RUNNING_CONTAINER_IDS; do
+      fn-stop-container "$APP" "$CID"
+    done
+  fi
 
-    "$DOCKER_BIN" container stop $DOCKER_STOP_TIME_ARG $DOKKU_APP_RUNNING_CONTAINER_IDS >/dev/null || true
+  # Stop all other containers related to the app
+  # fetch all containers with the com.dokku.app-name label
+  local DOKKU_APP_CIDS="$(docker container ls -q -f label=com.dokku.app-name="$APP")"
+  if [[ -n "$DOKKU_APP_CIDS" ]]; then
+    for CID in $DOKKU_APP_CIDS; do
+      fn-stop-container "$APP" "$CID"
+    done
   fi
 
   if [[ "$REMOVE_CONTAINERS" == "true" ]]; then
-    local DOKKU_APP_CIDS=$(get_app_container_ids "$APP")
+    local DOKKU_APP_CIDS="$(docker ps -q -f label=com.dokku.app-name="$APP")"
 
     if [[ -n "$DOKKU_APP_CIDS" ]]; then
       for CID in $DOKKU_APP_CIDS; do
@@ -36,6 +50,8 @@ trigger-scheduler-docker-local-scheduler-stop() {
       "$DOCKER_BIN" container rm --force $DOKKU_APP_CIDS &>/dev/null || true
     fi
   fi
+
+  plugn trigger scheduler-cron-write "$DOKKU_SCHEDULER" "$APP"
 }
 
 trigger-scheduler-docker-local-scheduler-stop "$@"

plugins/scheduler-docker-local/src/triggers/triggers.go

@@ -18,8 +18,9 @@ func main() {
 
 	var err error
 	switch trigger {
-	case "cron-write":
-		err = schedulerdockerlocal.TriggerCronWrite()
+	case "scheduler-cron-write":
+		scheduler := flag.Arg(0)
+		err = schedulerdockerlocal.TriggerSchedulerCronWrite(scheduler)
 	default:
 		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
 	}

plugins/scheduler-docker-local/triggers.go

@@ -1,6 +1,6 @@
 package schedulerdockerlocal
 
-// TriggerCronWrite force updates the cron file for all apps
-func TriggerCronWrite() error {
-	return writeCronEntries()
+// TriggerSchedulerCronWrite force updates the cron file for all apps
+func TriggerSchedulerCronWrite(scheduler string) error {
+	return writeCronEntries(scheduler)
 }

plugins/scheduler-k3s/k8s.go

@@ -940,3 +940,32 @@ func streamLogsFromRequest(ctx context.Context, request rest.ResponseWrapper, ou
 		}
 	}
 }
+
+// SuspendCronJobsInput contains all the information needed to suspend a Kubernetes cron job
+type SuspendCronJobsInput struct {
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// SuspendCronJobs suspends a Kubernetes cron job
+func (k KubernetesClient) SuspendCronJobs(ctx context.Context, input SuspendCronJobsInput) error {
+	cronJobs, err := k.Client.BatchV1().CronJobs(input.Namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: input.LabelSelector,
+	})
+	if err != nil {
+		return err
+	}
+
+	for _, cronJob := range cronJobs.Items {
+		cronJob.Spec.Suspend = ptr.To(true)
+		_, err := k.Client.BatchV1().CronJobs(input.Namespace).Update(ctx, &cronJob, metav1.UpdateOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

plugins/scheduler-k3s/triggers.go

@@ -266,7 +266,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 
 	workingDir := common.GetWorkingDir(appName, image)
 
-	allCronEntries, err := cron.FetchCronEntries(appName)
+	allCronEntries, err := cron.FetchCronEntries(cron.FetchCronEntriesInput{AppName: appName})
 	if err != nil {
 		return fmt.Errorf("Error fetching cron entries: %w", err)
 	}
@@ -1499,5 +1499,14 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 		}
 	}
 
+	// get all cronjobs for the app
+	err = clientset.SuspendCronJobs(ctx, SuspendCronJobsInput{
+		Namespace:     namespace,
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
+	})
+	if err != nil {
+		return fmt.Errorf("Error suspending cron jobs: %w", err)
+	}
+
 	return nil
 }

tests/unit/cron.bats

@@ -121,7 +121,7 @@ teardown() {
   echo "echo '@daily;/bin/true'" >/var/lib/dokku/plugins/enabled/cron-entries/cron-entries
   chmod +x /var/lib/dokku/plugins/enabled/cron-entries/cron-entries
 
-  run /bin/bash -c "dokku plugin:trigger cron-write"
+  run /bin/bash -c "dokku plugin:trigger scheduler-cron-write"
   echo "output: $output"
   echo "status: $status"
   assert_success
@@ -136,7 +136,7 @@ teardown() {
   echo "echo '@daily;/bin/true;/var/log/dokku/log.log'" >/var/lib/dokku/plugins/enabled/cron-entries/cron-entries
   chmod +x /var/lib/dokku/plugins/enabled/cron-entries/cron-entries
 
-  run /bin/bash -c "dokku plugin:trigger cron-write"
+  run /bin/bash -c "dokku plugin:trigger scheduler-cron-write"
   echo "output: $output"
   echo "status: $status"
   assert_success
@@ -151,7 +151,7 @@ teardown() {
   echo "[[ \$1 == 'docker-local' ]] && echo '@daily;/bin/true'" >/var/lib/dokku/plugins/enabled/cron-entries/cron-entries
   chmod +x /var/lib/dokku/plugins/enabled/cron-entries/cron-entries
 
-  run /bin/bash -c "dokku plugin:trigger cron-write"
+  run /bin/bash -c "dokku plugin:trigger scheduler-cron-write"
   echo "output: $output"
   echo "status: $status"
   assert_success
@@ -166,7 +166,7 @@ teardown() {
   echo "[[ \$1 == 'kubernetes' ]] && echo '@daily;/bin/true'" >/var/lib/dokku/plugins/enabled/cron-entries/cron-entries
   chmod +x /var/lib/dokku/plugins/enabled/cron-entries/cron-entries
 
-  run /bin/bash -c "dokku plugin:trigger cron-write"
+  run /bin/bash -c "dokku plugin:trigger scheduler-cron-write"
   echo "output: $output"
   echo "status: $status"
   assert_success