diff --git a/docs/appendices/0.25.0-migration-guide.md b/docs/appendices/0.25.0-migration-guide.md index ac70f9252..4e2ba0d82 100644 --- a/docs/appendices/0.25.0-migration-guide.md +++ b/docs/appendices/0.25.0-migration-guide.md @@ -10,3 +10,4 @@ - In previous versions of Dokku, the only way to specify a custom `Dockerfile` was to use the `docker-options` plugin to set the `--file` flag for a docker build. As of 0.25.0, the `builder-dockerfile:set` command should be used instead, as outlined in the [docs here](/docs/deployment/builders/dockerfiles.md#changingthe-dockerfile-location). Usage of the old method should be migrated to the new method. - The `--rm` and ``--rm-container` flags may be specified but no longer have any effect on `dokku run`. - The `--detach` flag is deprecated in favor of the `run:detached` command. +- The `DOKKU_SCALE` file is deprecated. Please see the [process management documentation](/docs/processes/process-management.md#manually-managing-process-scaling) for more information on it's replacement with the `formations` key of the `app.json` file. diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index a9d211c34..1181a9cff 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1800,6 +1800,50 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `ps-can-scale` + +- Description: Sets whether or not a user can scale an app with `ps:scale` +- Invoked by: +- Arguments: `$APP` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` +### `ps-current-scale` + +- Description: Prints out the current scale contents (process-type=quantity) delimited by newlines. +- Invoked by: +- Arguments: `$APP` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + +### `ps-set-scale` + +- Description: Sets the scale for an app based on a specified formation (process-type=quantity). Any unspecified process types will be left as is. +- Invoked by: +- Arguments: `$APP $SKIP_DEPLOY $CLEAR_EXISTING [$PROCESS_TUPLE...]` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + ### `receive-app` - Description: Allows you to customize what occurs when an app is received. Normally just triggers an app build. diff --git a/docs/processes/process-management.md b/docs/processes/process-management.md index 8b29bfb7d..98612d321 100644 --- a/docs/processes/process-management.md +++ b/docs/processes/process-management.md @@ -106,7 +106,7 @@ web: 1 #### Via CLI -> This functionality is disabled if the formation is managed via a file such as `DOKKU_SCALE`. +> This functionality is disabled if the formation is managed via the `formations` key of `app.json`. Dokku can also manage scaling itself via the `ps:scale` command. This command can be used to scale multiple process types at the same time. @@ -128,26 +128,34 @@ dokku ps:scale --skip-deploy node-js-app web=1 #### Manually managing process scaling -> Using a `DOKKU_SCALE` formation file disables the ability to use `ps:scale` for scaling. +> Using a `formations` key in an `app.json` file disables the ability to use `ps:scale` for scaling. -A `DOKKU_SCALE` file can be committed to the root of the pushed app repository, and must be within the built image artifact in the image's working directory as shown below. +An `app.json` file can be committed to the root of the pushed app repository, and must be within the built image artifact in the image's working directory as shown below. -- Buildpacks: `/app/DOKKU_SCALE` -- Dockerfile: `WORKDIR/DOKKU_SCALE` or `/DOKKU_SCALE` -- Docker Image: `WORKDIR/DOKKU_SCALE` or `/DOKKU_SCALE` +- Buildpacks: `/app/app.json` +- Dockerfile: `WORKDIR/app.json` or `/app.json` (if no working directory specified) +- Docker Image: `WORKDIR/app.json` or `/app.json` (if no working directory specified) -The `DOKKU_SCALE` file format is as follows: +The `formations` key should be specified as follows in the `app.json` file: ```Procfile -web=1 -worker=2 +{ + "formations": { + "web": { + "quantity": 1 + }, + "worker": { + "quantity": 4 + } + } +} ``` -Removing the file will result in Dokku respecting the `ps:scale` command for setting scale values. +Removing the file will result in Dokku respecting the `ps:scale` command for setting scale values. The values set via the `app.json` file from a previous deploy will be respected. #### The `web` process -For initial app deploys, Dokku will default to starting a single `web` process for each app. This process may be defined within the `Procfile` or as the `CMD` (for Dockerfile or Docker image deploys). Scaling of the `web` process - and others - may be managed via `ps:scale` or a `DOKKU_SCALE` formation file either before or after the initial deploy. +For initial app deploys, Dokku will default to starting a single `web` process for each app. This process may be defined within the `Procfile` or as the `CMD` (for Dockerfile or Docker image deploys). Scaling of the `web` process - and all other processes - may be managed via `ps:scale` or the `formations` key in the `app.json` file either before or after the initial deploy. There are also a few other exceptions for the `web` process. diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 8d064210d..6e3c4071b 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -20,8 +20,9 @@ var ( // AppJSON is a struct that represents an app.json file as understood by Dokku type AppJSON struct { - Cron []CronCommand `json:"cron"` - Scripts struct { + Cron []CronCommand `json:"cron"` + Formation map[string]Formation `json:"formation"` + Scripts struct { Dokku struct { Predeploy string `json:"predeploy"` Postdeploy string `json:"postdeploy"` @@ -36,6 +37,11 @@ type CronCommand struct { Schedule string `json:"schedule"` } +// Formation is a struct that represents the scale for a process from an app.json file +type Formation struct { + Quantity int `json:"quantity"` +} + // GetAppjsonDirectory returns the directory containing a given app's extracted app.json file func GetAppjsonDirectory(appName string) string { return filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "app-json", appName) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index e393f0133..1a3c6dd7f 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "github.com/dokku/dokku/plugins/common" @@ -79,24 +80,33 @@ func constructScript(command string, shell string, isHerokuishImage bool, isCnbI return []string{shell, "-c", strings.Join(script, " ")} } -// getPhaseScript extracts app.json from app image and returns the appropriate json key/value -func getPhaseScript(appName string, phase string) (string, error) { +func getAppJSON(appName string) (AppJSON, error) { if !common.FileExists(GetAppjsonPath(appName)) { - return "", nil + return AppJSON{}, nil } b, err := ioutil.ReadFile(GetAppjsonPath(appName)) if err != nil { - return "", fmt.Errorf("Cannot read app.json file: %v", err) + return AppJSON{}, fmt.Errorf("Cannot read app.json file: %v", err) } if strings.TrimSpace(string(b)) == "" { - return "", nil + return AppJSON{}, nil } var appJSON AppJSON if err = json.Unmarshal(b, &appJSON); err != nil { - return "", fmt.Errorf("Cannot parse app.json: %v", err) + return AppJSON{}, fmt.Errorf("Cannot parse app.json: %v", err) + } + + return appJSON, nil +} + +// getPhaseScript extracts app.json from app image and returns the appropriate json key/value +func getPhaseScript(appName string, phase string) (string, error) { + appJSON, err := getAppJSON(appName) + if err != nil { + return "", err } if phase == "heroku.postdeploy" { @@ -475,3 +485,96 @@ func refreshAppJSON(appName string, image string) error { common.CopyFromImage(appName, image, reportComputedAppjsonpath(appName), appjsonPath) return nil } + +func setScale(appName string, image string) error { + if err := injectDokkuScale(appName, image); err != nil { + return err + } + + appJSON, err := getAppJSON(appName) + if err != nil { + return err + } + + skipDeploy := true + clearExisting := false + args := []string{appName, strconv.FormatBool(skipDeploy), strconv.FormatBool(clearExisting)} + for processType, formation := range appJSON.Formation { + args = append(args, fmt.Sprintf("%s=%d", processType, formation.Quantity)) + } + + if len(args) == 3 { + return common.PlugnTrigger("ps-can-scale", []string{appName, "true"}...) + } + + if err := common.PlugnTrigger("ps-can-scale", []string{appName, "false"}...); err != nil { + return err + } + + return common.PlugnTrigger("ps-set-scale", args...) +} + +func injectDokkuScale(appName string, image string) error { + appJSON, err := getAppJSON(appName) + if err != nil { + return err + } + + baseDirectory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "app-json") + if !common.DirectoryExists(baseDirectory) { + return errors.New("Run 'dokku plugin:install' to ensure the correct directories exist") + } + + dokkuScaleFile := filepath.Join(baseDirectory, "DOKKU_SCALE") + previouslyExtracted := common.FileExists(dokkuScaleFile) + if previouslyExtracted { + os.Remove(dokkuScaleFile) + } + + common.CopyFromImage(appName, image, "DOKKU_SCALE", dokkuScaleFile) + + if !common.FileExists(dokkuScaleFile) { + return nil + } + + lines, err := common.FileToSlice(dokkuScaleFile) + if err != nil { + return err + } + + if appJSON.Formation == nil { + common.LogWarn("Deprecated: Injecting scale settings from DOKKU_SCALE file. Use the 'formation' key the app.json file to specify scaling instead of 'DOKKU_SCALE'.") + appJSON.Formation = make(map[string]Formation) + } else { + common.LogWarn("Deprecated: DOKKU_SCALE ignored in favor of 'formation' key in the app.json") + return nil + } + + for _, line := range lines { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + procParts := strings.SplitN(line, "=", 2) + if len(procParts) != 2 { + continue + } + + processType := procParts[0] + quantity, err := strconv.Atoi(procParts[1]) + if err != nil { + continue + } + + appJSON.Formation[processType] = Formation{ + Quantity: quantity, + } + } + + b, err := json.Marshal(appJSON) + if err != nil { + return err + } + + return ioutil.WriteFile(GetAppjsonPath(appName), b, 0644) +} diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index c3ccc05b5..708a5583b 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -64,6 +64,10 @@ func TriggerPreDeploy(appName string, imageTag string) error { return err } + if err := setScale(appName, image); err != nil { + return err + } + if common.PropertyGet("common", appName, "deployed") == "true" { return nil } diff --git a/plugins/checks/subcommands/run b/plugins/checks/subcommands/run index 3c5b747a1..8f1466957 100755 --- a/plugins/checks/subcommands/run +++ b/plugins/checks/subcommands/run @@ -7,15 +7,12 @@ source "$PLUGIN_AVAILABLE_PATH/checks/functions" check_all_processes() { local APP="$1" CHECK_PROC_TYPE="$2" - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" local VALID_CHECK_PROC_TYPE=false local line local PROC_TYPE local PROC_COUNT while read -r line || [[ -n "$line" ]]; do - [[ "$line" =~ ^#.* ]] && continue - line="$(strip_inline_comments "$line")" local PROC_TYPE=${line%%=*} local PROC_COUNT=${line#*=} @@ -30,7 +27,7 @@ check_all_processes() { fi check_process_type "$APP" "$PROC_TYPE" "$PROC_COUNT" - done <"$DOKKU_SCALE_FILE" + done < <(plugn trigger ps-current-scale "$APP") if [[ "$VALID_CHECK_PROC_TYPE" == "false" ]]; then dokku_log_fail "Invalid process type specified ($APP.$CHECK_PROC_TYPE)" diff --git a/plugins/common/common.go b/plugins/common/common.go index 47f3bbd13..bc1d25053 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -353,6 +353,23 @@ func ParseReportArgs(pluginName string, arguments []string) ([]string, string, e return osArgs, "", fmt.Errorf("%s:report command allows only a single flag", pluginName) } +// ParseScaleOutput allows golang plugins to properly parse the output of ps-current-scale +func ParseScaleOutput(b []byte) (map[string]int, error) { + scale := make(map[string]int) + + for _, line := range strings.Split(string(b), "\n") { + s := strings.Split(line, "=") + processType := s[0] + count, err := strconv.Atoi(s[1]) + if err != nil { + return scale, err + } + scale[processType] = count + } + + return scale, nil +} + // ReportSingleApp is an internal function that displays a report for an app func ReportSingleApp(reportType string, appName string, infoFlag string, infoFlags map[string]string, infoFlagKeys []string, format string, trimPrefix bool, uppercaseFirstCharacter bool) error { if format != "stdout" && infoFlag != "" { diff --git a/plugins/common/properties.go b/plugins/common/properties.go index 9b9f9fec7..8b7b58b41 100644 --- a/plugins/common/properties.go +++ b/plugins/common/properties.go @@ -173,6 +173,15 @@ func PropertyListAdd(pluginName string, appName string, property string, value s lines = append(lines, value) } + return PropertyListWrite(pluginName, appName, property, lines) +} + +// PropertyListWrite completely rewrites a list property +func PropertyListWrite(pluginName string, appName string, property string, values []string) error { + if err := propertyTouch(pluginName, appName, property); err != nil { + return err + } + propertyPath := getPropertyPath(pluginName, appName, property) file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600) if err != nil { @@ -180,7 +189,7 @@ func PropertyListAdd(pluginName string, appName string, property string, value s } w := bufio.NewWriter(file) - for _, line := range lines { + for _, line := range values { fmt.Fprintln(w, line) } if err = w.Flush(); err != nil { @@ -389,23 +398,7 @@ func PropertyListSet(pluginName string, appName string, property string, value s } } - propertyPath := getPropertyPath(pluginName, appName, property) - file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return err - } - - w := bufio.NewWriter(file) - for _, line := range lines { - fmt.Fprintln(w, line) - } - if err = w.Flush(); err != nil { - return fmt.Errorf("Unable to write %s config value %s.%s: %s", pluginName, appName, property, err.Error()) - } - - file.Chmod(0600) - SetPermissions(propertyPath, 0600) - return nil + return PropertyListWrite(pluginName, appName, property, lines) } // propertyTouch ensures a given application property file exists diff --git a/plugins/common/src/prop/prop.go b/plugins/common/src/prop/prop.go index c6ce2d0cf..9c06bb2bb 100644 --- a/plugins/common/src/prop/prop.go +++ b/plugins/common/src/prop/prop.go @@ -159,6 +159,16 @@ func main() { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } + case "lwrite": + _, args := common.ShiftString(flag.Args()) + appName, args := common.ShiftString(args) + property, args := common.ShiftString(args) + _, values := common.ShiftString(args) + err := common.PropertyListWrite(pluginName, appName, property, values) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } case "rpush": appName := flag.Arg(2) property := flag.Arg(3) diff --git a/plugins/network/network.go b/plugins/network/network.go index b7d77e8fd..64c772329 100644 --- a/plugins/network/network.go +++ b/plugins/network/network.go @@ -41,34 +41,25 @@ func BuildConfig(appName string) error { } appRoot := common.AppRoot(appName) - scaleFile := strings.Join([]string{appRoot, "DOKKU_SCALE"}, "/") - if !common.FileExists(scaleFile) { + s, err := common.PlugnTriggerOutput("ps-current-scale", []string{appName}...) + if err != nil { + return err + } + + scale, err := common.ParseScaleOutput(s) + if err != nil { + return err + } + + if len(scale) == 0 { return nil } image := common.GetAppImageName(appName, "", "") isHerokuishContainer := common.IsImageHerokuishBased(image, appName) common.LogInfo1(fmt.Sprintf("Ensuring network configuration is in sync for %s", appName)) - lines, err := common.FileToSlice(scaleFile) - if err != nil { - return err - } - - for _, line := range lines { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - procParts := strings.SplitN(line, "=", 2) - if len(procParts) != 2 { - continue - } - processType := procParts[0] - procCount, err := strconv.Atoi(procParts[1]) - if err != nil { - continue - } + for processType, procCount := range scale { containerIndex := 0 for containerIndex < procCount { containerIndex++ diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index d5092d1c9..cf356dd75 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -510,16 +510,13 @@ nginx_build_config() { PROXY_X_FORWARDED_PROTO="$PROXY_X_FORWARDED_PROTO" PROXY_X_FORWARDED_SSL="$PROXY_X_FORWARDED_SSL") - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" while read -r line || [[ -n "$line" ]]; do - [[ "$line" =~ ^#.* ]] && continue - line="$(strip_inline_comments "$line")" PROC_TYPE=${line%%=*} LISTENERS="$(plugn trigger network-get-listeners "$APP" "$PROC_TYPE" | xargs)" UPP_PROC_TYPE="${PROC_TYPE^^}" UPP_PROC_TYPE="${UPP_PROC_TYPE//-/_}" SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENERS=$LISTENERS") - done <"$DOKKU_SCALE_FILE" + done < <(plugn trigger ps-current-scale "$APP") if grep DOKKU_APP_LISTENERS "$NGINX_TEMPLATE"; then dokku_log_warn "Deprecated: Usage of DOKKU_APP_LISTENERS within nginx.conf.sigil templates is deprecated in favor of DOKKU_APP_WEB_LISTENERS" diff --git a/plugins/ps/.gitignore b/plugins/ps/.gitignore index 413fc555d..53d3a2be7 100644 --- a/plugins/ps/.gitignore +++ b/plugins/ps/.gitignore @@ -7,6 +7,7 @@ /pre-* /procfile-* /proxy-* +/ps-* /report /subcommands/* /triggers diff --git a/plugins/ps/Makefile b/plugins/ps/Makefile index 0761c2505..c8610988d 100644 --- a/plugins/ps/Makefile +++ b/plugins/ps/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/inspect subcommands/rebuild subcommands/report subcommands/restart subcommands/restore subcommands/retire subcommands/scale subcommands/set subcommands/start subcommands/stop -TRIGGERS = triggers/app-restart triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-extract triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/report +TRIGGERS = triggers/app-restart triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-extract triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/ps-can-scale triggers/ps-current-scale triggers/ps-set-scale triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = ps diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index f6a47bbe8..991fdaa64 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -16,7 +16,8 @@ import ( ) func canScaleApp(appName string) bool { - return !common.FileExists(getScalefileExtractedPath(appName)) + canScale := common.PropertyGetDefault("ps", appName, "can-scale", "true") + return common.ToBool(canScale) } func extractProcfile(appName, image string) error { @@ -46,64 +47,6 @@ func extractProcfile(appName, image string) error { return nil } -func extractOrGenerateScalefile(appName string, image string) error { - destination := getScalefilePath(appName) - extracted := getScalefileExtractedPath(appName) - previouslyExtracted := common.FileExists(extracted) - - if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", extracted); err != nil { - if previouslyExtracted { - os.Remove(destination) - } - os.Remove(extracted) - } else if err := common.CopyFile(extracted, destination); err != nil { - return err - } - - if common.FileExists(destination) { - common.LogInfo1Quiet("DOKKU_SCALE file exists") - return updateScalefile(appName, make(map[string]int)) - } - - common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") - if err := updateScalefile(appName, make(map[string]int)); err != nil { - common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) - return err - } - return nil -} - -func generateScalefile(appName string) error { - destination := getScalefilePath(appName) - procfilePath := getProcfilePath(appName) - content := []string{"web=1"} - if !common.FileExists(procfilePath) { - return common.WriteSliceToFile(destination, content) - } - - lines, err := common.FileToSlice(procfilePath) - if err != nil { - return common.WriteSliceToFile(destination, content) - } - - content = []string{} - for _, line := range lines { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - parts := strings.Split(line, ":") - count := 0 - if parts[0] == "web" { - count = 1 - } - - content = append(content, fmt.Sprintf("%s=%d", parts[0], count)) - } - - return common.WriteSliceToFile(destination, content) -} - func getProcessStatus(appName string) map[string]string { statuses := make(map[string]string) containerFiles := common.ListFilesWithPrefix(common.AppRoot(appName), "CONTAINER.") @@ -182,19 +125,6 @@ func getRunningState(appName string) string { return strings.Split(strings.TrimSpace(string(b[:])), " ")[1] } -func getScalefilePath(appName string) string { - return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE") -} - -func getScalefileExtractedPath(appName string) string { - return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") -} - -func hasScaleFile(appName string) bool { - scalefilePath := getScalefilePath(appName) - return common.FileExists(scalefilePath) -} - func hasProcfile(appName string) bool { procfilePath := getProcfilePath(appName) return common.FileExists(procfilePath) @@ -219,25 +149,28 @@ func isValidRestartPolicy(policy string) bool { return strings.HasPrefix(policy, "on-failure:") } -func parseProcessTuples(processTuples []string) (map[string]int, error) { - scale := make(map[string]int) +func parseProcessTuples(processTuples []string) (FormationSlice, error) { + formations := FormationSlice{} for _, processTuple := range processTuples { s := strings.Split(processTuple, "=") if len(s) == 1 { - return scale, fmt.Errorf("Missing count for process type %s", processTuple) + return formations, fmt.Errorf("Missing count for process type %s", processTuple) } processType := s[0] - count, err := strconv.Atoi(s[1]) + quantity, err := strconv.Atoi(s[1]) if err != nil { - return scale, fmt.Errorf("Invalid count for process type %s", s[0]) + return formations, fmt.Errorf("Invalid count for process type %s", s[0]) } - scale[processType] = count + formations = append(formations, &Formation{ + ProcessType: processType, + Quantity: quantity, + }) } - return scale, nil + return formations, nil } func processesInProcfile(procfilePath string) (map[string]bool, error) { @@ -265,41 +198,14 @@ func processesInProcfile(procfilePath string) (map[string]bool, error) { return processes, nil } -func readScaleFile(appName string) (map[string]int, error) { - scale := make(map[string]int) - if !hasScaleFile(appName) { - return scale, errors.New("No DOKKU_SCALE file found") - } - - scalefilePath := getScalefilePath(appName) - lines, err := common.FileToSlice(scalefilePath) +func getFormations(appName string) (FormationSlice, error) { + formations := FormationSlice{} + processTuples, err := common.PropertyListGet("ps", appName, "scale") if err != nil { - return scale, err + return formations, err } - for i, line := range lines { - s := strings.Split(line, "=") - if len(s) != 2 { - common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i)) - continue - } - - processType := s[0] - count, err := strconv.Atoi(s[1]) - if err != nil { - common.LogWarn(fmt.Sprintf("Invalid count on line %d, skipping", i)) - continue - } - - scale[processType] = count - } - - if len(scale) == 0 { - common.LogWarn("No valid entries found in scale file, defaulting to web=1") - scale["web"] = 1 - } - - return scale, nil + return parseProcessTuples(processTuples) } func removeProcfile(appName string) error { @@ -328,8 +234,7 @@ func restorePrep() error { } func scaleReport(appName string) error { - scalefilePath := getScalefilePath(appName) - lines, err := common.FileToSlice(scalefilePath) + formations, err := getFormations(appName) if err != nil { return err } @@ -346,9 +251,9 @@ func scaleReport(appName string) error { content = append(content, "proctype=qty", "--------=---") } - sort.Strings(lines) - for _, line := range lines { - content = append(content, line) + sort.Sort(formations) + for _, formation := range formations { + content = append(content, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) } for _, line := range content { @@ -359,21 +264,20 @@ func scaleReport(appName string) error { return nil } -func scaleSet(appName string, skipDeploy bool, processTuples []string) error { - if !canScaleApp(appName) { - return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) - } - - scale, err := parseProcessTuples(processTuples) +func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error { + formations, err := parseProcessTuples(processTuples) if err != nil { return err } - common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) - if err := updateScalefile(appName, scale); err != nil { + if err := updateScale(appName, clearExisting, formations); err != nil { return err } + if skipDeploy { + return nil + } + if !common.IsDeployed(appName) { return nil } @@ -383,59 +287,75 @@ func scaleSet(appName string, skipDeploy bool, processTuples []string) error { return err } - if skipDeploy { - return nil - } - return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) } -func updateScalefile(appName string, scaleUpdates map[string]int) error { - if !hasScaleFile(appName) { - if err := generateScalefile(appName); err != nil { +func updateScale(appName string, clearExisting bool, formationUpdates FormationSlice) error { + formations := FormationSlice{} + if !clearExisting { + processTuples, err := common.PropertyListGet("ps", appName, "scale") + if err != nil { return err } - } - scale, err := readScaleFile(appName) - if err != nil { - return err + formations, err = parseProcessTuples(processTuples) + if err != nil { + return err + } } procfilePath := getProcfilePath(appName) procfileExists := hasProcfile(appName) validProcessTypes := make(map[string]bool) if procfileExists { + var err error validProcessTypes, err = processesInProcfile(procfilePath) if err != nil { return err } } - for processType, count := range scaleUpdates { - if procfileExists && !validProcessTypes[processType] && count != 0 { - return fmt.Errorf("%s is not a valid process name to scale up", processType) + foundProcessTypes := map[string]bool{} + updatedFormation := FormationSlice{} + for _, formation := range formationUpdates { + if procfileExists && !validProcessTypes[formation.ProcessType] && formation.Quantity != 0 { + return fmt.Errorf("%s is not a valid process name to scale up", formation.ProcessType) } - scale[processType] = count + + foundProcessTypes[formation.ProcessType] = true + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: formation.ProcessType, + Quantity: formation.Quantity, + }) + } + + for _, formation := range formations { + if foundProcessTypes[formation.ProcessType] { + continue + } + + foundProcessTypes[formation.ProcessType] = true + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: formation.ProcessType, + Quantity: formation.Quantity, + }) } for processType := range validProcessTypes { - count, ok := scale[processType] - if !ok { - count = 0 + if foundProcessTypes[processType] { + continue } - scale[processType] = count + + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: processType, + Quantity: 0, + }) } - content := []string{} - for processType, count := range scale { - content = append(content, fmt.Sprintf("%s=%d", processType, count)) + values := []string{} + for _, formation := range updatedFormation { + values = append(values, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) } - scalefilePath := getScalefilePath(appName) - if err := common.WriteSliceToFile(scalefilePath, content); err != nil { - return err - } - - return nil + return common.PropertyListWrite("ps", appName, "scale", values) } diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 10699d9e5..8b2ecddcf 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -24,6 +24,27 @@ var ( } ) +// Formation contains scaling information for a given process type +type Formation struct { + ProcessType string `json:"process_type"` + Quantity int `json:"quantity"` +} + +// FormationSlice contains a slice of Formations that can be sorted +type FormationSlice []*Formation + +func (d FormationSlice) Len() int { + return len(d) +} + +func (d FormationSlice) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} + +func (d FormationSlice) Less(i, j int) bool { + return d[i].ProcessType < d[j].ProcessType +} + // Rebuild rebuilds app from base image func Rebuild(appName string) error { return common.PlugnTrigger("receive-app", []string{appName}...) diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index 35d3bc675..9593c7d6e 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -75,6 +75,19 @@ func main() { case "procfile-remove": appName := flag.Arg(0) err = ps.TriggerProcfileRemove(appName) + case "ps-can-scale": + appName := flag.Arg(0) + canScale := common.ToBool(flag.Arg(1)) + err = ps.TriggerPsCanScale(appName, canScale) + case "ps-current-scale": + appName := flag.Arg(0) + err = ps.TriggerPsCurrentScale(appName) + case "ps-set-scale": + appName := flag.Arg(0) + appName, args := common.ShiftString(flag.Args()) + skipDeploy, args := common.ShiftString(args) + clearExisting, processTuples := common.ShiftString(args) + err = ps.TriggerPsSetScale(appName, common.ToBool(skipDeploy), common.ToBool(clearExisting), processTuples) case "report": appName := flag.Arg(0) err = ps.ReportSingleApp(appName, "", "") diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 9c675f872..1ba4703d2 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" dockeroptions "github.com/dokku/dokku/plugins/docker-options" @@ -120,21 +121,16 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error return err } - procfilePath := getProcfilePath(appName) - if !hasScaleFile(appName) || common.FileExists(procfilePath) { - update := func() error { - return updateScalefile(appName, make(map[string]int)) - } - if err := common.SuppressOutput(update); err != nil { - return err - } - } - if len(processTuples) == 0 { return scaleReport(appName) } - return scaleSet(appName, skipDeploy, processTuples) + if !canScaleApp(appName) { + return fmt.Errorf("App %s contains an app.json file with a formations key and cannot be manually scaled", appName) + } + + common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) + return scaleSet(appName, skipDeploy, false, processTuples) } // CommandSet sets or clears a ps property for an app diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index af6fd2cca..4723ee9af 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -5,6 +5,8 @@ import ( "os" "path" "path/filepath" + "sort" + "strconv" "strings" sh "github.com/codeskyblue/go-sh" @@ -89,6 +91,30 @@ func TriggerInstall() error { } } + for _, appName := range apps { + dokkuScaleFile := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE") + if common.FileExists(dokkuScaleFile) { + processTuples, err := common.FileToSlice(dokkuScaleFile) + if err != nil { + return err + } + + if err := scaleSet(appName, true, false, processTuples); err != nil { + return err + } + + os.Remove(dokkuScaleFile) + } + + dokkuScaleExtracted := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") + if common.FileExists(dokkuScaleExtracted) { + if err := common.PropertyWrite("ps", appName, "can-scale", strconv.FormatBool(false)); err != nil { + return err + } + os.Remove(dokkuScaleExtracted) + } + } + return nil } @@ -149,7 +175,13 @@ func TriggerPostCreate(appName string) error { return err } - return updateScalefile(appName, make(map[string]int)) + formations := FormationSlice{ + &Formation{ + ProcessType: "web", + Quantity: 1, + }, + } + return updateScale(appName, false, formations) } // TriggerPostDelete destroys the ps properties for a given app container @@ -190,7 +222,7 @@ func TriggerPostStop(appName string) error { }) } -// TriggerPreDeploy ensures an app has an up to date scale file +// TriggerPreDeploy ensures an app has an up to date scale parameters func TriggerPreDeploy(appName string, imageTag string) error { image, err := common.GetDeployingAppImageName(appName, imageTag, "") if err != nil { @@ -205,7 +237,8 @@ func TriggerPreDeploy(appName string, imageTag string) error { return err } - if err := extractOrGenerateScalefile(appName, image); err != nil { + if err := updateScale(appName, false, FormationSlice{}); err != nil { + common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) return err } @@ -263,3 +296,31 @@ func TriggerProcfileGetCommand(appName string, processType string, port int) err func TriggerProcfileRemove(appName string) error { return removeProcfile(appName) } + +// TriggerPsCanScale sets whether or not a user can scale an app with ps:scale +func TriggerPsCanScale(appName string, canScale bool) error { + return common.PropertyWrite("ps", appName, "can-scale", strconv.FormatBool(canScale)) +} + +// TriggerPsCurrentScale prints out the current scale contents (process-type=quantity) delimited by newlines +func TriggerPsCurrentScale(appName string) error { + formations, err := getFormations(appName) + if err != nil { + return err + } + + sort.Sort(formations) + lines := []string{} + for _, formation := range formations { + lines = append(lines, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) + } + + fmt.Print(strings.Join(lines, "\n")) + + return nil +} + +// TriggerPsSetScale configures the scale parameters for a given app +func TriggerPsSetScale(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error { + return scaleSet(appName, skipDeploy, clearExisting, processTuples) +} diff --git a/plugins/scheduler-docker-local/core-post-deploy b/plugins/scheduler-docker-local/core-post-deploy index 4edb1f656..d29f72b02 100755 --- a/plugins/scheduler-docker-local/core-post-deploy +++ b/plugins/scheduler-docker-local/core-post-deploy @@ -16,7 +16,7 @@ trigger-scheduler-docker-local-core-post-deploy() { fi dokku_log_info1 "Renaming containers" - local PROCTYPES="$(grep -v -E "^#" "$DOKKU_ROOT/$APP/DOKKU_SCALE" | awk -F '=' '{ print $1 }' | xargs)" + local PROCTYPES="$(plugn trigger ps-current-scale "$APP" | awk -F '=' '{ print $1 }' | xargs)" local CONTAINER_FILES="$(find "$DOKKU_ROOT/$APP" -maxdepth 1 -name "CONTAINER.*" -printf "%f\n" 2>/dev/null | sort -t . -k 3 -n | xargs)" local CONTAINER_FILE diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index fb6e4dafe..ab3a1c281 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -29,7 +29,6 @@ trigger-scheduler-docker-local-scheduler-deploy() { local IMAGE_SOURCE_TYPE="dockerfile" [[ "$DOKKU_HEROKUISH" == "true" ]] && IMAGE_SOURCE_TYPE="herokuish" [[ "$DOKKU_CNB" == "true" ]] && IMAGE_SOURCE_TYPE="pack" - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" local oldids=$(get_app_container_ids "$APP") DOKKU_NETWORK_BIND_ALL="$(plugn trigger network-get-property "$APP" bind-all-interfaces)" @@ -51,8 +50,6 @@ trigger-scheduler-docker-local-scheduler-deploy() { local PROC_COUNT local CONTAINER_INDEX while read -r line || [[ -n "$line" ]]; do - [[ "$line" =~ ^#.* ]] && continue - line="$(strip_inline_comments "$line")" PROC_TYPE=${line%%=*} PROC_COUNT=${line#*=} CONTAINER_INDEX=1 @@ -178,7 +175,7 @@ trigger-scheduler-docker-local-scheduler-deploy() { cd "$DOKKU_ROOT/$APP" find . -maxdepth 1 -name "$container_state_filetype.$PROC_TYPE.*" -printf "%f\n" | sort -t . -k 3 -n | tail -n +$CONTAINER_IDX_OFFSET | xargs rm -f done - done <"$DOKKU_SCALE_FILE" + done < <(plugn trigger ps-current-scale "$APP") dokku_log_info1 "Running post-deploy" plugn trigger core-post-deploy "$APP" "$port" "$ipaddr" "$IMAGE_TAG" diff --git a/tests/unit/nginx-vhosts_2.bats b/tests/unit/nginx-vhosts_2.bats index ed985806f..7769a058d 100644 --- a/tests/unit/nginx-vhosts_2.bats +++ b/tests/unit/nginx-vhosts_2.bats @@ -36,9 +36,14 @@ teardown() { assert_ssl_domain "wildcard2.dokku.me" } -@test "(nginx-vhosts) nginx:build-config (wildcard SSL and unrelated domain)" { +@test "(nginx-vhosts) nginx:build-config (wildcard SSL and unrelated domain) 1" { destroy_app TEST_APP="${TEST_APP}.example.com" + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + setup_test_tls wildcard deploy_app nodejs-express dokku@dokku.me:$TEST_APP run /bin/bash -c "dokku nginx:show-config $TEST_APP | grep -e '*.dokku.me' | wc -l" diff --git a/tests/unit/ps-general.bats b/tests/unit/ps-general.bats index cd237b151..7c8a5bc34 100644 --- a/tests/unit/ps-general.bats +++ b/tests/unit/ps-general.bats @@ -55,7 +55,7 @@ EOF assert_output "node worker.js" } -@test "(ps:scale) update DOKKU_SCALE from Procfile" { +@test "(ps:scale) update formations from Procfile" { local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM