From 9d0bc930a67a9972622deaf793c2d8fde7abe31a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 22 Mar 2021 00:24:34 -0400 Subject: [PATCH 01/37] fix: handle empty lines and comments in DOKKU_SCALE file --- plugins/ps/functions.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index f6a47bbe8..a03632466 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -278,6 +278,10 @@ func readScaleFile(appName string) (map[string]int, error) { } for i, line := range lines { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + s := strings.Split(line, "=") if len(s) != 2 { common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i)) From 6d9520e59959049a39b8bf9b35480bd73f251ae2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 22 Mar 2021 00:24:56 -0400 Subject: [PATCH 02/37] feat: add support for formation key in app.json file Still needs triggers for ps-can-scale and ps-scale-set. Refs #4500 --- plugins/app-json/appjson.go | 9 +++++-- plugins/app-json/functions.go | 47 ++++++++++++++++++++++++++++++----- plugins/app-json/triggers.go | 4 +++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 8d064210d..4d8aad640 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -20,8 +20,9 @@ var ( // AppJSON is a struct that represents an app.json file as understood by Dokku type AppJSON struct { - Cron []CronCommand `json:"cron"` - Scripts struct { + Cron []CronCommand `json:"cron"` + Formation map[string]Formation `json:"formation"` + Scripts struct { Dokku struct { Predeploy string `json:"predeploy"` Postdeploy string `json:"postdeploy"` @@ -36,6 +37,10 @@ type CronCommand struct { Schedule string `json:"schedule"` } +type Formation struct { + Quantity int `json:"quantity"` +} + // GetAppjsonDirectory returns the directory containing a given app's extracted app.json file func GetAppjsonDirectory(appName string) string { return filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "app-json", appName) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index e393f0133..e370382a3 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -79,24 +79,33 @@ func constructScript(command string, shell string, isHerokuishImage bool, isCnbI return []string{shell, "-c", strings.Join(script, " ")} } -// getPhaseScript extracts app.json from app image and returns the appropriate json key/value -func getPhaseScript(appName string, phase string) (string, error) { +func getAppJson(appName string) (AppJSON, error) { if !common.FileExists(GetAppjsonPath(appName)) { - return "", nil + return AppJSON{}, nil } b, err := ioutil.ReadFile(GetAppjsonPath(appName)) if err != nil { - return "", fmt.Errorf("Cannot read app.json file: %v", err) + return AppJSON{}, fmt.Errorf("Cannot read app.json file: %v", err) } if strings.TrimSpace(string(b)) == "" { - return "", nil + return AppJSON{}, nil } var appJSON AppJSON if err = json.Unmarshal(b, &appJSON); err != nil { - return "", fmt.Errorf("Cannot parse app.json: %v", err) + return AppJSON{}, fmt.Errorf("Cannot parse app.json: %v", err) + } + + return AppJSON{}, nil +} + +// getPhaseScript extracts app.json from app image and returns the appropriate json key/value +func getPhaseScript(appName string, phase string) (string, error) { + appJSON, err := getAppJson(appName) + if err != nil { + return "", err } if phase == "heroku.postdeploy" { @@ -475,3 +484,29 @@ func refreshAppJSON(appName string, image string) error { common.CopyFromImage(appName, image, reportComputedAppjsonpath(appName), appjsonPath) return nil } + +func setScale(appName string, image string) error { + appJSON, err := getAppJson(appName) + if err != nil { + return err + } + + args := []string{appName} + for processType, formation := range appJSON.Formation { + args = append(args, fmt.Sprintf("%s=%d", processType, formation.Quantity)) + } + + if len(args) == 1 { + return common.PlugnTrigger("ps-can-scale", []string{appName, "true"}...) + } + + if err := common.PlugnTrigger("ps-can-scale", []string{appName, "false"}...); err != nil { + return err + } + + if err := common.PlugnTrigger("ps-scale-set", args...); err != nil { + return err + } + + return nil +} diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index c3ccc05b5..708a5583b 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -64,6 +64,10 @@ func TriggerPreDeploy(appName string, imageTag string) error { return err } + if err := setScale(appName, image); err != nil { + return err + } + if common.PropertyGet("common", appName, "deployed") == "true" { return nil } From bd2aa1bd437e540e46b559b7ed20a0a02fe37036 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 22 Mar 2021 00:26:45 -0400 Subject: [PATCH 03/37] feat: migrate the DOKKU_SCALE property during pre-deploy to app.json --- plugins/app-json/functions.go | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index e370382a3..00c6172ed 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "github.com/dokku/dokku/plugins/common" @@ -486,6 +487,10 @@ func refreshAppJSON(appName string, image string) error { } func setScale(appName string, image string) error { + if err := injectDokkuScale(appName, image); err != nil { + return err + } + appJSON, err := getAppJson(appName) if err != nil { return err @@ -510,3 +515,63 @@ func setScale(appName string, image string) error { return nil } + +func injectDokkuScale(appName string, image string) error { + appJSON, err := getAppJson(appName) + if err != nil { + return err + } + + baseDirectory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "app-json") + if !common.DirectoryExists(baseDirectory) { + return errors.New("Run 'dokku plugin:install' to ensure the correct directories exist") + } + + dokkuScaleFile := filepath.Join(baseDirectory, "DOKKU_SCALE") + previouslyExtracted := common.FileExists(dokkuScaleFile) + if previouslyExtracted { + os.Remove(dokkuScaleFile) + } + + if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", dokkuScaleFile); err != nil { + return err + } + + if !common.FileExists(dokkuScaleFile) { + return nil + } + + common.LogWarn("Deprecated: Use the 'formation' key in app.json to specify scaling instead of DOKKU_SCALE") + lines, err := common.FileToSlice(dokkuScaleFile) + if err != nil { + return err + } + + for _, line := range lines { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + procParts := strings.SplitN(line, "=", 2) + if len(procParts) != 2 { + continue + } + + processType := procParts[0] + procCount, err := strconv.Atoi(procParts[1]) + if err != nil { + continue + } + + appJSON.Formation[processType] = Formation{ + Quantity: procCount, + } + } + + b, err := json.Marshal(appJSON) + if err != nil { + return err + } + + return ioutil.WriteFile(dokkuScaleFile, b, 0644) +} From c6d99722eb060c0760943164ffa3f03a7bfb53d2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 17 Apr 2021 00:29:15 -0400 Subject: [PATCH 04/37] fix: correct stickler errors --- plugins/app-json/appjson.go | 1 + plugins/app-json/functions.go | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 4d8aad640..6e3c4071b 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -37,6 +37,7 @@ type CronCommand struct { Schedule string `json:"schedule"` } +// Formation is a struct that represents the scale for a process from an app.json file type Formation struct { Quantity int `json:"quantity"` } diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 00c6172ed..c47bf468c 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -80,7 +80,7 @@ func constructScript(command string, shell string, isHerokuishImage bool, isCnbI return []string{shell, "-c", strings.Join(script, " ")} } -func getAppJson(appName string) (AppJSON, error) { +func getAppJSON(appName string) (AppJSON, error) { if !common.FileExists(GetAppjsonPath(appName)) { return AppJSON{}, nil } @@ -104,7 +104,7 @@ func getAppJson(appName string) (AppJSON, error) { // getPhaseScript extracts app.json from app image and returns the appropriate json key/value func getPhaseScript(appName string, phase string) (string, error) { - appJSON, err := getAppJson(appName) + appJSON, err := getAppJSON(appName) if err != nil { return "", err } @@ -491,7 +491,7 @@ func setScale(appName string, image string) error { return err } - appJSON, err := getAppJson(appName) + appJSON, err := getAppJSON(appName) if err != nil { return err } @@ -517,7 +517,7 @@ func setScale(appName string, image string) error { } func injectDokkuScale(appName string, image string) error { - appJSON, err := getAppJson(appName) + appJSON, err := getAppJSON(appName) if err != nil { return err } From 67317f88dfbd969764e3c8f19d4a67180b5e1c61 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 17 Apr 2021 00:30:48 -0400 Subject: [PATCH 05/37] fix: do not force fetching of the DOKKU_SCALE file --- plugins/app-json/functions.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index c47bf468c..32ec5a276 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -533,9 +533,7 @@ func injectDokkuScale(appName string, image string) error { os.Remove(dokkuScaleFile) } - if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", dokkuScaleFile); err != nil { - return err - } + common.CopyFromImage(appName, image, "DOKKU_SCALE", dokkuScaleFile) if !common.FileExists(dokkuScaleFile) { return nil From b9e30e7e196a52cf82a55e0716fa73c689881fb4 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 31 Jul 2021 22:23:19 -0400 Subject: [PATCH 06/37] fix: remove redundant if check --- plugins/app-json/functions.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 32ec5a276..7d3b8f5f2 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -509,11 +509,7 @@ func setScale(appName string, image string) error { return err } - if err := common.PlugnTrigger("ps-scale-set", args...); err != nil { - return err - } - - return nil + return common.PlugnTrigger("ps-scale-set", args...) } func injectDokkuScale(appName string, image string) error { From 772e4286f2e14c57642ed77c682fbf6b77e3e6dc Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 31 Jul 2021 22:44:05 -0400 Subject: [PATCH 07/37] fix: variable reference --- plugins/app-json/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 7d3b8f5f2..1e56ca065 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -99,7 +99,7 @@ func getAppJSON(appName string) (AppJSON, error) { return AppJSON{}, fmt.Errorf("Cannot parse app.json: %v", err) } - return AppJSON{}, nil + return appJSON, nil } // getPhaseScript extracts app.json from app image and returns the appropriate json key/value From 3fe10efd09b408690b97eee32664803615c32642 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 31 Jul 2021 23:00:37 -0400 Subject: [PATCH 08/37] fix: correct issue with assignment to nil map --- plugins/app-json/functions.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 1e56ca065..20d7586e6 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -541,6 +541,10 @@ func injectDokkuScale(appName string, image string) error { return err } + if appJSON.Formation == nil { + appJSON.Formation = make(map[string]Formation) + } + for _, line := range lines { if line == "" || strings.HasPrefix(line, "#") { continue From bc49a2e124a86cb75da32a967ed9ee76ffc4a0ef Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 00:30:31 -0400 Subject: [PATCH 09/37] fix: ignore DOKKU_SCALE file if app.json has formations key --- plugins/app-json/functions.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 20d7586e6..6958046a9 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -535,14 +535,17 @@ func injectDokkuScale(appName string, image string) error { return nil } - common.LogWarn("Deprecated: Use the 'formation' key in app.json to specify scaling instead of DOKKU_SCALE") lines, err := common.FileToSlice(dokkuScaleFile) if err != nil { return err } if appJSON.Formation == nil { + common.LogWarn("Deprecated: Use the 'formation' key in app.json to specify scaling instead of DOKKU_SCALE") appJSON.Formation = make(map[string]Formation) + } else { + common.LogWarn("Deprecated: DOKKU_SCALE ignored in favor of 'formation' key in app.json") + return nil } for _, line := range lines { From fee6773ef3408f4f891d1909958a68a6621b8ae1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 00:30:53 -0400 Subject: [PATCH 10/37] docs: add deprecation note to migration guide Also add documentation for formations key. --- docs/appendices/0.25.0-migration-guide.md | 1 + docs/processes/process-management.md | 28 +++++++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/appendices/0.25.0-migration-guide.md b/docs/appendices/0.25.0-migration-guide.md index ac70f9252..4e2ba0d82 100644 --- a/docs/appendices/0.25.0-migration-guide.md +++ b/docs/appendices/0.25.0-migration-guide.md @@ -10,3 +10,4 @@ - In previous versions of Dokku, the only way to specify a custom `Dockerfile` was to use the `docker-options` plugin to set the `--file` flag for a docker build. As of 0.25.0, the `builder-dockerfile:set` command should be used instead, as outlined in the [docs here](/docs/deployment/builders/dockerfiles.md#changingthe-dockerfile-location). Usage of the old method should be migrated to the new method. - The `--rm` and ``--rm-container` flags may be specified but no longer have any effect on `dokku run`. - The `--detach` flag is deprecated in favor of the `run:detached` command. +- The `DOKKU_SCALE` file is deprecated. Please see the [process management documentation](/docs/processes/process-management.md#manually-managing-process-scaling) for more information on it's replacement with the `formations` key of the `app.json` file. diff --git a/docs/processes/process-management.md b/docs/processes/process-management.md index 8b29bfb7d..576dc9297 100644 --- a/docs/processes/process-management.md +++ b/docs/processes/process-management.md @@ -106,7 +106,7 @@ web: 1 #### Via CLI -> This functionality is disabled if the formation is managed via a file such as `DOKKU_SCALE`. +> This functionality is disabled if the formation is managed via the `formations` key of `app.json`. Dokku can also manage scaling itself via the `ps:scale` command. This command can be used to scale multiple process types at the same time. @@ -128,22 +128,30 @@ dokku ps:scale --skip-deploy node-js-app web=1 #### Manually managing process scaling -> Using a `DOKKU_SCALE` formation file disables the ability to use `ps:scale` for scaling. +> Using a `formations` key in an `app.json` file disables the ability to use `ps:scale` for scaling. -A `DOKKU_SCALE` file can be committed to the root of the pushed app repository, and must be within the built image artifact in the image's working directory as shown below. +An `app.json` file can be committed to the root of the pushed app repository, and must be within the built image artifact in the image's working directory as shown below. -- Buildpacks: `/app/DOKKU_SCALE` -- Dockerfile: `WORKDIR/DOKKU_SCALE` or `/DOKKU_SCALE` -- Docker Image: `WORKDIR/DOKKU_SCALE` or `/DOKKU_SCALE` +- Buildpacks: `/app/app.json` +- Dockerfile: `WORKDIR/app.json` or `/app.json` (if no working directory specified) +- Docker Image: `WORKDIR/app.json` or `/app.json` (if no working directory specified) -The `DOKKU_SCALE` file format is as follows: +The `formations` key should be specified as follows in the `app.json` file: ```Procfile -web=1 -worker=2 +{ + "formations": { + "web": { + "quantity": 1 + }, + "worker": { + "quantity": 4 + } + } +} ``` -Removing the file will result in Dokku respecting the `ps:scale` command for setting scale values. +Removing the file will result in Dokku respecting the `ps:scale` command for setting scale values. The values set via the `app.json` file from a previous deploy will be respected. #### The `web` process From 31f3c429ab28b23e3db1e83a5be2c78070eeb6e2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 00:33:36 -0400 Subject: [PATCH 11/37] docs: fix DOKKU_SCALE reference --- docs/processes/process-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/processes/process-management.md b/docs/processes/process-management.md index 576dc9297..98612d321 100644 --- a/docs/processes/process-management.md +++ b/docs/processes/process-management.md @@ -155,7 +155,7 @@ Removing the file will result in Dokku respecting the `ps:scale` command for set #### The `web` process -For initial app deploys, Dokku will default to starting a single `web` process for each app. This process may be defined within the `Procfile` or as the `CMD` (for Dockerfile or Docker image deploys). Scaling of the `web` process - and others - may be managed via `ps:scale` or a `DOKKU_SCALE` formation file either before or after the initial deploy. +For initial app deploys, Dokku will default to starting a single `web` process for each app. This process may be defined within the `Procfile` or as the `CMD` (for Dockerfile or Docker image deploys). Scaling of the `web` process - and all other processes - may be managed via `ps:scale` or the `formations` key in the `app.json` file either before or after the initial deploy. There are also a few other exceptions for the `web` process. From c62bff300209897755f2bcc0430909e44bd518af Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 01:21:46 -0400 Subject: [PATCH 12/37] refactor: hide dokku scale file parsing inside ps plugin --- plugins/ps/functions.go | 27 ++++++++++++++++++++++----- plugins/ps/ps.go | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index a03632466..f2d91baae 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -306,6 +306,24 @@ func readScaleFile(appName string) (map[string]int, error) { return scale, nil } +func getFormations(appName string) (FormationSlice, error) { + scale := FormationSlice{} + data, err := readScaleFile(appName) + if err != nil { + return scale, err + } + + for processType, quantity := range data { + f := Formation{ + ProcessType: processType, + Quantity: quantity, + } + scale = append(scale, &f) + } + + return scale, nil +} + func removeProcfile(appName string) error { procfile := getProcfilePath(appName) if !common.FileExists(procfile) { @@ -332,8 +350,7 @@ func restorePrep() error { } func scaleReport(appName string) error { - scalefilePath := getScalefilePath(appName) - lines, err := common.FileToSlice(scalefilePath) + formations, err := getFormations(appName) if err != nil { return err } @@ -350,9 +367,9 @@ func scaleReport(appName string) error { content = append(content, "proctype=qty", "--------=---") } - sort.Strings(lines) - for _, line := range lines { - content = append(content, line) + sort.Sort(formations) + for _, formation := range formations { + content = append(content, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) } for _, line := range content { diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 10699d9e5..9d524c109 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -24,6 +24,25 @@ var ( } ) +type FormationSlice []*Formation + +type Formation struct { + ProcessType string `json:"process_type"` + Quantity int `json:"quantity"` +} + +func (d FormationSlice) Len() int { + return len(d) +} + +func (d FormationSlice) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} + +func (d FormationSlice) Less(i, j int) bool { + return d[i].ProcessType < d[j].ProcessType +} + // Rebuild rebuilds app from base image func Rebuild(appName string) error { return common.PlugnTrigger("receive-app", []string{appName}...) From 213f7d224c420470ab08983aa767271ca0ec3ebe Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 01:43:35 -0400 Subject: [PATCH 13/37] refactor: use ps-current-scale instead of parsing DOKKU_SCALE file when iterating over deployed processes --- docs/development/plugin-triggers.md | 15 ++++++ plugins/checks/subcommands/run | 3 +- plugins/network/network.go | 49 +++++++++++-------- plugins/nginx-vhosts/functions | 3 +- plugins/ps/Makefile | 2 +- plugins/ps/src/triggers/triggers.go | 3 ++ plugins/ps/triggers.go | 16 ++++++ .../scheduler-docker-local/core-post-deploy | 2 +- .../scheduler-docker-local/scheduler-deploy | 3 +- 9 files changed, 67 insertions(+), 29 deletions(-) diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index a9d211c34..1f2c8c6a5 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1800,6 +1800,21 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `ps-current-scale` + +- Description: Prints out the current scale contents (process-type=quantity) delimited by newlines. +- Invoked by: +- Arguments: `$APP` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + ### `receive-app` - Description: Allows you to customize what occurs when an app is received. Normally just triggers an app build. diff --git a/plugins/checks/subcommands/run b/plugins/checks/subcommands/run index 3c5b747a1..b774c7977 100755 --- a/plugins/checks/subcommands/run +++ b/plugins/checks/subcommands/run @@ -7,7 +7,6 @@ source "$PLUGIN_AVAILABLE_PATH/checks/functions" check_all_processes() { local APP="$1" CHECK_PROC_TYPE="$2" - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" local VALID_CHECK_PROC_TYPE=false local line @@ -30,7 +29,7 @@ check_all_processes() { fi check_process_type "$APP" "$PROC_TYPE" "$PROC_COUNT" - done <"$DOKKU_SCALE_FILE" + done <"$(plugn trigger ps-current-scale "$APP")" if [[ "$VALID_CHECK_PROC_TYPE" == "false" ]]; then dokku_log_fail "Invalid process type specified ($APP.$CHECK_PROC_TYPE)" diff --git a/plugins/network/network.go b/plugins/network/network.go index b7d77e8fd..39d14efb1 100644 --- a/plugins/network/network.go +++ b/plugins/network/network.go @@ -34,6 +34,22 @@ var ( } ) +func parseScaleOutput(b []byte) (map[string]int, error) { + scale := make(map[string]int) + + for _, line := range strings.Split(string(b), "\n") { + s := strings.Split(line, "=") + processType := s[0] + count, err := strconv.Atoi(s[1]) + if err != nil { + return scale, err + } + scale[processType] = count + } + + return scale, nil +} + // BuildConfig builds network config files func BuildConfig(appName string) error { if !common.IsDeployed(appName) { @@ -41,34 +57,25 @@ func BuildConfig(appName string) error { } appRoot := common.AppRoot(appName) - scaleFile := strings.Join([]string{appRoot, "DOKKU_SCALE"}, "/") - if !common.FileExists(scaleFile) { + s, err := common.PlugnTriggerOutput("ps-current-scale", []string{appName}...) + if err != nil { + return err + } + + scale, err := parseScaleOutput(s) + if err != nil { + return err + } + + if len(scale) == 0 { return nil } image := common.GetAppImageName(appName, "", "") isHerokuishContainer := common.IsImageHerokuishBased(image, appName) common.LogInfo1(fmt.Sprintf("Ensuring network configuration is in sync for %s", appName)) - lines, err := common.FileToSlice(scaleFile) - if err != nil { - return err - } - - for _, line := range lines { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - procParts := strings.SplitN(line, "=", 2) - if len(procParts) != 2 { - continue - } - processType := procParts[0] - procCount, err := strconv.Atoi(procParts[1]) - if err != nil { - continue - } + for processType, procCount := range scale { containerIndex := 0 for containerIndex < procCount { containerIndex++ diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index d5092d1c9..d1f776eda 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -510,7 +510,6 @@ nginx_build_config() { PROXY_X_FORWARDED_PROTO="$PROXY_X_FORWARDED_PROTO" PROXY_X_FORWARDED_SSL="$PROXY_X_FORWARDED_SSL") - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" while read -r line || [[ -n "$line" ]]; do [[ "$line" =~ ^#.* ]] && continue line="$(strip_inline_comments "$line")" @@ -519,7 +518,7 @@ nginx_build_config() { UPP_PROC_TYPE="${PROC_TYPE^^}" UPP_PROC_TYPE="${UPP_PROC_TYPE//-/_}" SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENERS=$LISTENERS") - done <"$DOKKU_SCALE_FILE" + done <"$(plugn trigger ps-current-scale "$APP")" if grep DOKKU_APP_LISTENERS "$NGINX_TEMPLATE"; then dokku_log_warn "Deprecated: Usage of DOKKU_APP_LISTENERS within nginx.conf.sigil templates is deprecated in favor of DOKKU_APP_WEB_LISTENERS" diff --git a/plugins/ps/Makefile b/plugins/ps/Makefile index 0761c2505..e2a68b39a 100644 --- a/plugins/ps/Makefile +++ b/plugins/ps/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/inspect subcommands/rebuild subcommands/report subcommands/restart subcommands/restore subcommands/retire subcommands/scale subcommands/set subcommands/start subcommands/stop -TRIGGERS = triggers/app-restart triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-extract triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/report +TRIGGERS = triggers/app-restart triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-extract triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/ps-current-scale triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = ps diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index 35d3bc675..161521ea9 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -75,6 +75,9 @@ func main() { case "procfile-remove": appName := flag.Arg(0) err = ps.TriggerProcfileRemove(appName) + case "ps-current-scale": + appName := flag.Arg(0) + err = ps.TriggerPsCurrentScale(appName) case "report": appName := flag.Arg(0) err = ps.ReportSingleApp(appName, "", "") diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index af6fd2cca..b5044659d 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -5,6 +5,7 @@ import ( "os" "path" "path/filepath" + "sort" "strings" sh "github.com/codeskyblue/go-sh" @@ -263,3 +264,18 @@ func TriggerProcfileGetCommand(appName string, processType string, port int) err func TriggerProcfileRemove(appName string) error { return removeProcfile(appName) } + +// TriggerPsCurrentScale prints out the current scale contents (process-type=quantity) delimited by newlines +func TriggerPsCurrentScale(appName string) error { + formations, err := getFormations(appName) + if err != nil { + return err + } + + sort.Sort(formations) + for _, formation := range formations { + fmt.Printf("%s=%d\n", formation.ProcessType, formation.Quantity) + } + + return nil +} diff --git a/plugins/scheduler-docker-local/core-post-deploy b/plugins/scheduler-docker-local/core-post-deploy index 4edb1f656..d29f72b02 100755 --- a/plugins/scheduler-docker-local/core-post-deploy +++ b/plugins/scheduler-docker-local/core-post-deploy @@ -16,7 +16,7 @@ trigger-scheduler-docker-local-core-post-deploy() { fi dokku_log_info1 "Renaming containers" - local PROCTYPES="$(grep -v -E "^#" "$DOKKU_ROOT/$APP/DOKKU_SCALE" | awk -F '=' '{ print $1 }' | xargs)" + local PROCTYPES="$(plugn trigger ps-current-scale "$APP" | awk -F '=' '{ print $1 }' | xargs)" local CONTAINER_FILES="$(find "$DOKKU_ROOT/$APP" -maxdepth 1 -name "CONTAINER.*" -printf "%f\n" 2>/dev/null | sort -t . -k 3 -n | xargs)" local CONTAINER_FILE diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index fb6e4dafe..13d17345a 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -29,7 +29,6 @@ trigger-scheduler-docker-local-scheduler-deploy() { local IMAGE_SOURCE_TYPE="dockerfile" [[ "$DOKKU_HEROKUISH" == "true" ]] && IMAGE_SOURCE_TYPE="herokuish" [[ "$DOKKU_CNB" == "true" ]] && IMAGE_SOURCE_TYPE="pack" - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" local oldids=$(get_app_container_ids "$APP") DOKKU_NETWORK_BIND_ALL="$(plugn trigger network-get-property "$APP" bind-all-interfaces)" @@ -178,7 +177,7 @@ trigger-scheduler-docker-local-scheduler-deploy() { cd "$DOKKU_ROOT/$APP" find . -maxdepth 1 -name "$container_state_filetype.$PROC_TYPE.*" -printf "%f\n" | sort -t . -k 3 -n | tail -n +$CONTAINER_IDX_OFFSET | xargs rm -f done - done <"$DOKKU_SCALE_FILE" + done <"$(plugn trigger ps-current-scale "$APP")" dokku_log_info1 "Running post-deploy" plugn trigger core-post-deploy "$APP" "$port" "$ipaddr" "$IMAGE_TAG" From 550a0bd07b8de0b7942cf1fa8f8196773121662c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 01:53:46 -0400 Subject: [PATCH 14/37] fix: properly parse input from ps-current-scale --- plugins/checks/subcommands/run | 2 +- plugins/nginx-vhosts/functions | 2 +- plugins/scheduler-docker-local/scheduler-deploy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/checks/subcommands/run b/plugins/checks/subcommands/run index b774c7977..29121fd3a 100755 --- a/plugins/checks/subcommands/run +++ b/plugins/checks/subcommands/run @@ -29,7 +29,7 @@ check_all_processes() { fi check_process_type "$APP" "$PROC_TYPE" "$PROC_COUNT" - done <"$(plugn trigger ps-current-scale "$APP")" + done < <(plugn trigger ps-current-scale "$APP") if [[ "$VALID_CHECK_PROC_TYPE" == "false" ]]; then dokku_log_fail "Invalid process type specified ($APP.$CHECK_PROC_TYPE)" diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index d1f776eda..19baf6a9a 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -518,7 +518,7 @@ nginx_build_config() { UPP_PROC_TYPE="${PROC_TYPE^^}" UPP_PROC_TYPE="${UPP_PROC_TYPE//-/_}" SIGIL_PARAMS+=("DOKKU_APP_${UPP_PROC_TYPE}_LISTENERS=$LISTENERS") - done <"$(plugn trigger ps-current-scale "$APP")" + done < <(plugn trigger ps-current-scale "$APP") if grep DOKKU_APP_LISTENERS "$NGINX_TEMPLATE"; then dokku_log_warn "Deprecated: Usage of DOKKU_APP_LISTENERS within nginx.conf.sigil templates is deprecated in favor of DOKKU_APP_WEB_LISTENERS" diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index 13d17345a..c83de4573 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -177,7 +177,7 @@ trigger-scheduler-docker-local-scheduler-deploy() { cd "$DOKKU_ROOT/$APP" find . -maxdepth 1 -name "$container_state_filetype.$PROC_TYPE.*" -printf "%f\n" | sort -t . -k 3 -n | tail -n +$CONTAINER_IDX_OFFSET | xargs rm -f done - done <"$(plugn trigger ps-current-scale "$APP")" + done < <(plugn trigger ps-current-scale "$APP") dokku_log_info1 "Running post-deploy" plugn trigger core-post-deploy "$APP" "$port" "$ipaddr" "$IMAGE_TAG" From 33bf6f59b3bfc93b85e377ba40e0158a62d6b23e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 15:31:59 -0400 Subject: [PATCH 15/37] fix: do not output extra newlines --- plugins/ps/triggers.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index b5044659d..4e0fa47ed 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -273,9 +273,12 @@ func TriggerPsCurrentScale(appName string) error { } sort.Sort(formations) + lines := []string{} for _, formation := range formations { - fmt.Printf("%s=%d\n", formation.ProcessType, formation.Quantity) + lines = append(lines, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) } + fmt.Print(strings.Join(lines, "\n")) + return nil } From 1c98b48e5de034e1cd3d8272afee7e1ac244800d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 15:32:08 -0400 Subject: [PATCH 16/37] fix: add trigger to gitignore --- plugins/ps/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/ps/.gitignore b/plugins/ps/.gitignore index 413fc555d..53d3a2be7 100644 --- a/plugins/ps/.gitignore +++ b/plugins/ps/.gitignore @@ -7,6 +7,7 @@ /pre-* /procfile-* /proxy-* +/ps-* /report /subcommands/* /triggers From a62f04fa7ebd2e8b0fe50db781f01d9ffde82417 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 15:47:55 -0400 Subject: [PATCH 17/37] docs: add godoc --- plugins/ps/ps.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 9d524c109..8b2ecddcf 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -24,13 +24,15 @@ var ( } ) -type FormationSlice []*Formation - +// Formation contains scaling information for a given process type type Formation struct { ProcessType string `json:"process_type"` Quantity int `json:"quantity"` } +// FormationSlice contains a slice of Formations that can be sorted +type FormationSlice []*Formation + func (d FormationSlice) Len() int { return len(d) } From 302537efd28ff32109d540069b68a2a1e7d4ae36 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:07:26 -0400 Subject: [PATCH 18/37] refactor: skip image fetching when deploys are skipped --- plugins/ps/functions.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index f2d91baae..3a1b6abd9 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -395,6 +395,10 @@ func scaleSet(appName string, skipDeploy bool, processTuples []string) error { return err } + if skipDeploy { + return nil + } + if !common.IsDeployed(appName) { return nil } @@ -404,10 +408,6 @@ func scaleSet(appName string, skipDeploy bool, processTuples []string) error { return err } - if skipDeploy { - return nil - } - return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) } From 247698fea6b60f52f2f2ef5f2df41d3eee5ab151 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:07:55 -0400 Subject: [PATCH 19/37] refactor: move can-scale check out to command If the internals call ps-set-scale, we should always respect it. --- plugins/ps/functions.go | 4 ---- plugins/ps/subcommands.go | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 3a1b6abd9..e62a4064d 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -381,10 +381,6 @@ func scaleReport(appName string) error { } func scaleSet(appName string, skipDeploy bool, processTuples []string) error { - if !canScaleApp(appName) { - return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) - } - scale, err := parseProcessTuples(processTuples) if err != nil { return err diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 9c675f872..ed15efcf7 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -134,6 +134,10 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error return scaleReport(appName) } + if !canScaleApp(appName) { + return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) + } + return scaleSet(appName, skipDeploy, processTuples) } From 46d333df4b92bb586674f5d042c1bf98c591c28d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:11:09 -0400 Subject: [PATCH 20/37] feat: implement ps-set-scale trigger --- docs/development/plugin-triggers.md | 15 +++++++++++++++ plugins/app-json/functions.go | 4 ++-- plugins/ps/src/triggers/triggers.go | 5 +++++ plugins/ps/triggers.go | 4 ++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 1f2c8c6a5..95a552aa1 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1815,6 +1815,21 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `ps-set-scale` + +- Description: Sets the scale for an app based on a specified formation (process-type=quantity). Any unspecified process types will be left as is. +- Invoked by: +- Arguments: `$APP $SKIP_DEPLOY [$PROCESS_TUPLE...]` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + ### `receive-app` - Description: Allows you to customize what occurs when an app is received. Normally just triggers an app build. diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 6958046a9..fdde679c2 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -496,7 +496,7 @@ func setScale(appName string, image string) error { return err } - args := []string{appName} + args := []string{appName, "true"} for processType, formation := range appJSON.Formation { args = append(args, fmt.Sprintf("%s=%d", processType, formation.Quantity)) } @@ -509,7 +509,7 @@ func setScale(appName string, image string) error { return err } - return common.PlugnTrigger("ps-scale-set", args...) + return common.PlugnTrigger("ps-set-scale", args...) } func injectDokkuScale(appName string, image string) error { diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index 161521ea9..e3a365fbd 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -75,6 +75,11 @@ func main() { case "procfile-remove": appName := flag.Arg(0) err = ps.TriggerProcfileRemove(appName) + case "ps-set-scale": + appName := flag.Arg(0) + appName, args := common.ShiftString(flag.Args()) + skipDeploy, processTuples := common.ShiftString(args) + err = ps.TriggerPsSetScale(appName, common.ToBool(skipDeploy), processTuples) case "ps-current-scale": appName := flag.Arg(0) err = ps.TriggerPsCurrentScale(appName) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 4e0fa47ed..5b5d6280b 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -282,3 +282,7 @@ func TriggerPsCurrentScale(appName string) error { return nil } + +func TriggerPsSetScale(appName string, skipDeploy bool, processTuples []string) error { + return scaleSet(appName, skipDeploy, processTuples) +} From 5ad5a29a00eeec16b8f09b9cca3b50bc6c7c4b78 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:14:12 -0400 Subject: [PATCH 21/37] docs: add godoc --- plugins/ps/triggers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 5b5d6280b..dfae9f020 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -283,6 +283,7 @@ func TriggerPsCurrentScale(appName string) error { return nil } +// TriggerPsSetScale configures the scale parameters for a given app func TriggerPsSetScale(appName string, skipDeploy bool, processTuples []string) error { return scaleSet(appName, skipDeploy, processTuples) } From ef262c26e8e227358829547c48f0d67d785c4d29 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:20:19 -0400 Subject: [PATCH 22/37] feat: override all existing scale parameters when calling ps-set-scale --- plugins/ps/functions.go | 21 +++++++++++++-------- plugins/ps/subcommands.go | 4 ++-- plugins/ps/triggers.go | 4 ++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index e62a4064d..f47bb9c64 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -62,11 +62,11 @@ func extractOrGenerateScalefile(appName string, image string) error { if common.FileExists(destination) { common.LogInfo1Quiet("DOKKU_SCALE file exists") - return updateScalefile(appName, make(map[string]int)) + return updateScalefile(appName, false, make(map[string]int)) } common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") - if err := updateScalefile(appName, make(map[string]int)); err != nil { + if err := updateScalefile(appName, false, make(map[string]int)); err != nil { common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) return err } @@ -380,14 +380,14 @@ func scaleReport(appName string) error { return nil } -func scaleSet(appName string, skipDeploy bool, processTuples []string) error { +func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error { scale, err := parseProcessTuples(processTuples) if err != nil { return err } common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) - if err := updateScalefile(appName, scale); err != nil { + if err := updateScalefile(appName, clearExisting, scale); err != nil { return err } @@ -407,22 +407,27 @@ func scaleSet(appName string, skipDeploy bool, processTuples []string) error { return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) } -func updateScalefile(appName string, scaleUpdates map[string]int) error { +func updateScalefile(appName string, clearExisting bool, scaleUpdates map[string]int) error { if !hasScaleFile(appName) { if err := generateScalefile(appName); err != nil { return err } } - scale, err := readScaleFile(appName) - if err != nil { - return err + scale := make(map[string]int) + if !clearExisting { + var err error + scale, err = readScaleFile(appName) + if err != nil { + return err + } } procfilePath := getProcfilePath(appName) procfileExists := hasProcfile(appName) validProcessTypes := make(map[string]bool) if procfileExists { + var err error validProcessTypes, err = processesInProcfile(procfilePath) if err != nil { return err diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index ed15efcf7..2fb208ba5 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -123,7 +123,7 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error procfilePath := getProcfilePath(appName) if !hasScaleFile(appName) || common.FileExists(procfilePath) { update := func() error { - return updateScalefile(appName, make(map[string]int)) + return updateScalefile(appName, false, make(map[string]int)) } if err := common.SuppressOutput(update); err != nil { return err @@ -138,7 +138,7 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) } - return scaleSet(appName, skipDeploy, processTuples) + return scaleSet(appName, skipDeploy, false, processTuples) } // CommandSet sets or clears a ps property for an app diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index dfae9f020..e152e447d 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -150,7 +150,7 @@ func TriggerPostCreate(appName string) error { return err } - return updateScalefile(appName, make(map[string]int)) + return updateScalefile(appName, false, make(map[string]int)) } // TriggerPostDelete destroys the ps properties for a given app container @@ -285,5 +285,5 @@ func TriggerPsCurrentScale(appName string) error { // TriggerPsSetScale configures the scale parameters for a given app func TriggerPsSetScale(appName string, skipDeploy bool, processTuples []string) error { - return scaleSet(appName, skipDeploy, processTuples) + return scaleSet(appName, skipDeploy, true, processTuples) } From 70eb751868ea5acfd17446f53d2721a122b498ae Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:26:29 -0400 Subject: [PATCH 23/37] fix: correct ps-can-scale guard --- plugins/app-json/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index fdde679c2..7ca481fc4 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -501,7 +501,7 @@ func setScale(appName string, image string) error { args = append(args, fmt.Sprintf("%s=%d", processType, formation.Quantity)) } - if len(args) == 1 { + if len(args) == 2 { return common.PlugnTrigger("ps-can-scale", []string{appName, "true"}...) } From bd4621070c4de940618d6a8922b2c4fbbb01e4f0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:27:13 -0400 Subject: [PATCH 24/37] feat: implement ps-can-scale --- plugins/ps/functions.go | 3 ++- plugins/ps/src/triggers/triggers.go | 10 +++++++--- plugins/ps/triggers.go | 6 ++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index f47bb9c64..b318c86df 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -16,7 +16,8 @@ import ( ) func canScaleApp(appName string) bool { - return !common.FileExists(getScalefileExtractedPath(appName)) + canScale := common.PropertyGetDefault("ps", appName, "can-scale", "true") + return common.ToBool(canScale) } func extractProcfile(appName, image string) error { diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index e3a365fbd..bad436b7b 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -75,14 +75,18 @@ func main() { case "procfile-remove": appName := flag.Arg(0) err = ps.TriggerProcfileRemove(appName) + case "ps-can-scale": + appName := flag.Arg(0) + canScale := common.ToBool(flag.Arg(1)) + err = ps.TriggerPsCanScale(appName, canScale) + case "ps-current-scale": + appName := flag.Arg(0) + err = ps.TriggerPsCurrentScale(appName) case "ps-set-scale": appName := flag.Arg(0) appName, args := common.ShiftString(flag.Args()) skipDeploy, processTuples := common.ShiftString(args) err = ps.TriggerPsSetScale(appName, common.ToBool(skipDeploy), processTuples) - case "ps-current-scale": - appName := flag.Arg(0) - err = ps.TriggerPsCurrentScale(appName) case "report": appName := flag.Arg(0) err = ps.ReportSingleApp(appName, "", "") diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index e152e447d..9de576f28 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -6,6 +6,7 @@ import ( "path" "path/filepath" "sort" + "strconv" "strings" sh "github.com/codeskyblue/go-sh" @@ -265,6 +266,11 @@ func TriggerProcfileRemove(appName string) error { return removeProcfile(appName) } +// TriggerPsCanScale sets whether or not a user can scale an app with ps:scale +func TriggerPsCanScale(appName string, canScale bool) error { + return common.PropertyWrite("ps", appName, "can-scale", strconv.FormatBool(canScale)) +} + // TriggerPsCurrentScale prints out the current scale contents (process-type=quantity) delimited by newlines func TriggerPsCurrentScale(appName string) error { formations, err := getFormations(appName) From a1dcc2c17375bc59de8000549cd86af8312d20ed Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:29:50 -0400 Subject: [PATCH 25/37] refactor: move ParseScaleOutput to common.go so that it can be used by all plugins --- plugins/common/common.go | 17 +++++++++++++++++ plugins/network/network.go | 18 +----------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index 47f3bbd13..bc1d25053 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -353,6 +353,23 @@ func ParseReportArgs(pluginName string, arguments []string) ([]string, string, e return osArgs, "", fmt.Errorf("%s:report command allows only a single flag", pluginName) } +// ParseScaleOutput allows golang plugins to properly parse the output of ps-current-scale +func ParseScaleOutput(b []byte) (map[string]int, error) { + scale := make(map[string]int) + + for _, line := range strings.Split(string(b), "\n") { + s := strings.Split(line, "=") + processType := s[0] + count, err := strconv.Atoi(s[1]) + if err != nil { + return scale, err + } + scale[processType] = count + } + + return scale, nil +} + // ReportSingleApp is an internal function that displays a report for an app func ReportSingleApp(reportType string, appName string, infoFlag string, infoFlags map[string]string, infoFlagKeys []string, format string, trimPrefix bool, uppercaseFirstCharacter bool) error { if format != "stdout" && infoFlag != "" { diff --git a/plugins/network/network.go b/plugins/network/network.go index 39d14efb1..64c772329 100644 --- a/plugins/network/network.go +++ b/plugins/network/network.go @@ -34,22 +34,6 @@ var ( } ) -func parseScaleOutput(b []byte) (map[string]int, error) { - scale := make(map[string]int) - - for _, line := range strings.Split(string(b), "\n") { - s := strings.Split(line, "=") - processType := s[0] - count, err := strconv.Atoi(s[1]) - if err != nil { - return scale, err - } - scale[processType] = count - } - - return scale, nil -} - // BuildConfig builds network config files func BuildConfig(appName string) error { if !common.IsDeployed(appName) { @@ -62,7 +46,7 @@ func BuildConfig(appName string) error { return err } - scale, err := parseScaleOutput(s) + scale, err := common.ParseScaleOutput(s) if err != nil { return err } From 8fc0cb67ec5dc7175a1da283cd5f009fd733875b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 16:33:42 -0400 Subject: [PATCH 26/37] chore: drop unnecessary line stripping Since this comes out of ps-current-scale, we can ensure the contents are always valid. --- plugins/checks/subcommands/run | 2 -- plugins/nginx-vhosts/functions | 2 -- plugins/scheduler-docker-local/scheduler-deploy | 2 -- 3 files changed, 6 deletions(-) diff --git a/plugins/checks/subcommands/run b/plugins/checks/subcommands/run index 29121fd3a..8f1466957 100755 --- a/plugins/checks/subcommands/run +++ b/plugins/checks/subcommands/run @@ -13,8 +13,6 @@ check_all_processes() { local PROC_TYPE local PROC_COUNT while read -r line || [[ -n "$line" ]]; do - [[ "$line" =~ ^#.* ]] && continue - line="$(strip_inline_comments "$line")" local PROC_TYPE=${line%%=*} local PROC_COUNT=${line#*=} diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index 19baf6a9a..cf356dd75 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -511,8 +511,6 @@ nginx_build_config() { PROXY_X_FORWARDED_SSL="$PROXY_X_FORWARDED_SSL") while read -r line || [[ -n "$line" ]]; do - [[ "$line" =~ ^#.* ]] && continue - line="$(strip_inline_comments "$line")" PROC_TYPE=${line%%=*} LISTENERS="$(plugn trigger network-get-listeners "$APP" "$PROC_TYPE" | xargs)" UPP_PROC_TYPE="${PROC_TYPE^^}" diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index c83de4573..ab3a1c281 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -50,8 +50,6 @@ trigger-scheduler-docker-local-scheduler-deploy() { local PROC_COUNT local CONTAINER_INDEX while read -r line || [[ -n "$line" ]]; do - [[ "$line" =~ ^#.* ]] && continue - line="$(strip_inline_comments "$line")" PROC_TYPE=${line%%=*} PROC_COUNT=${line#*=} CONTAINER_INDEX=1 From 401683a3059350dedeabd7e74942ed68f113637b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 17:47:04 -0400 Subject: [PATCH 27/37] refactor: write DOKKU_SCALE contents to scale property - remove extraction of DOKKU_SCALE - migrate to property system - rewrite internals to use a FormationSlice instead of a map --- plugins/common/properties.go | 29 ++--- plugins/common/src/prop/prop.go | 10 ++ plugins/ps/functions.go | 211 ++++++++------------------------ plugins/ps/subcommands.go | 14 +-- plugins/ps/triggers.go | 31 ++++- tests/unit/ps-general.bats | 2 +- 6 files changed, 105 insertions(+), 192 deletions(-) diff --git a/plugins/common/properties.go b/plugins/common/properties.go index 9b9f9fec7..8b7b58b41 100644 --- a/plugins/common/properties.go +++ b/plugins/common/properties.go @@ -173,6 +173,15 @@ func PropertyListAdd(pluginName string, appName string, property string, value s lines = append(lines, value) } + return PropertyListWrite(pluginName, appName, property, lines) +} + +// PropertyListWrite completely rewrites a list property +func PropertyListWrite(pluginName string, appName string, property string, values []string) error { + if err := propertyTouch(pluginName, appName, property); err != nil { + return err + } + propertyPath := getPropertyPath(pluginName, appName, property) file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600) if err != nil { @@ -180,7 +189,7 @@ func PropertyListAdd(pluginName string, appName string, property string, value s } w := bufio.NewWriter(file) - for _, line := range lines { + for _, line := range values { fmt.Fprintln(w, line) } if err = w.Flush(); err != nil { @@ -389,23 +398,7 @@ func PropertyListSet(pluginName string, appName string, property string, value s } } - propertyPath := getPropertyPath(pluginName, appName, property) - file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return err - } - - w := bufio.NewWriter(file) - for _, line := range lines { - fmt.Fprintln(w, line) - } - if err = w.Flush(); err != nil { - return fmt.Errorf("Unable to write %s config value %s.%s: %s", pluginName, appName, property, err.Error()) - } - - file.Chmod(0600) - SetPermissions(propertyPath, 0600) - return nil + return PropertyListWrite(pluginName, appName, property, lines) } // propertyTouch ensures a given application property file exists diff --git a/plugins/common/src/prop/prop.go b/plugins/common/src/prop/prop.go index c6ce2d0cf..9c06bb2bb 100644 --- a/plugins/common/src/prop/prop.go +++ b/plugins/common/src/prop/prop.go @@ -159,6 +159,16 @@ func main() { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } + case "lwrite": + _, args := common.ShiftString(flag.Args()) + appName, args := common.ShiftString(args) + property, args := common.ShiftString(args) + _, values := common.ShiftString(args) + err := common.PropertyListWrite(pluginName, appName, property, values) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } case "rpush": appName := flag.Arg(2) property := flag.Arg(3) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index b318c86df..134f97f32 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -47,64 +47,6 @@ func extractProcfile(appName, image string) error { return nil } -func extractOrGenerateScalefile(appName string, image string) error { - destination := getScalefilePath(appName) - extracted := getScalefileExtractedPath(appName) - previouslyExtracted := common.FileExists(extracted) - - if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", extracted); err != nil { - if previouslyExtracted { - os.Remove(destination) - } - os.Remove(extracted) - } else if err := common.CopyFile(extracted, destination); err != nil { - return err - } - - if common.FileExists(destination) { - common.LogInfo1Quiet("DOKKU_SCALE file exists") - return updateScalefile(appName, false, make(map[string]int)) - } - - common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") - if err := updateScalefile(appName, false, make(map[string]int)); err != nil { - common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) - return err - } - return nil -} - -func generateScalefile(appName string) error { - destination := getScalefilePath(appName) - procfilePath := getProcfilePath(appName) - content := []string{"web=1"} - if !common.FileExists(procfilePath) { - return common.WriteSliceToFile(destination, content) - } - - lines, err := common.FileToSlice(procfilePath) - if err != nil { - return common.WriteSliceToFile(destination, content) - } - - content = []string{} - for _, line := range lines { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - parts := strings.Split(line, ":") - count := 0 - if parts[0] == "web" { - count = 1 - } - - content = append(content, fmt.Sprintf("%s=%d", parts[0], count)) - } - - return common.WriteSliceToFile(destination, content) -} - func getProcessStatus(appName string) map[string]string { statuses := make(map[string]string) containerFiles := common.ListFilesWithPrefix(common.AppRoot(appName), "CONTAINER.") @@ -183,19 +125,6 @@ func getRunningState(appName string) string { return strings.Split(strings.TrimSpace(string(b[:])), " ")[1] } -func getScalefilePath(appName string) string { - return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE") -} - -func getScalefileExtractedPath(appName string) string { - return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") -} - -func hasScaleFile(appName string) bool { - scalefilePath := getScalefilePath(appName) - return common.FileExists(scalefilePath) -} - func hasProcfile(appName string) bool { procfilePath := getProcfilePath(appName) return common.FileExists(procfilePath) @@ -220,25 +149,28 @@ func isValidRestartPolicy(policy string) bool { return strings.HasPrefix(policy, "on-failure:") } -func parseProcessTuples(processTuples []string) (map[string]int, error) { - scale := make(map[string]int) +func parseProcessTuples(processTuples []string) (FormationSlice, error) { + formations := FormationSlice{} for _, processTuple := range processTuples { s := strings.Split(processTuple, "=") if len(s) == 1 { - return scale, fmt.Errorf("Missing count for process type %s", processTuple) + return formations, fmt.Errorf("Missing count for process type %s", processTuple) } processType := s[0] - count, err := strconv.Atoi(s[1]) + quantity, err := strconv.Atoi(s[1]) if err != nil { - return scale, fmt.Errorf("Invalid count for process type %s", s[0]) + return formations, fmt.Errorf("Invalid count for process type %s", s[0]) } - scale[processType] = count + formations = append(formations, &Formation{ + ProcessType: processType, + Quantity: quantity, + }) } - return scale, nil + return formations, nil } func processesInProcfile(procfilePath string) (map[string]bool, error) { @@ -266,63 +198,14 @@ func processesInProcfile(procfilePath string) (map[string]bool, error) { return processes, nil } -func readScaleFile(appName string) (map[string]int, error) { - scale := make(map[string]int) - if !hasScaleFile(appName) { - return scale, errors.New("No DOKKU_SCALE file found") - } - - scalefilePath := getScalefilePath(appName) - lines, err := common.FileToSlice(scalefilePath) - if err != nil { - return scale, err - } - - for i, line := range lines { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - s := strings.Split(line, "=") - if len(s) != 2 { - common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i)) - continue - } - - processType := s[0] - count, err := strconv.Atoi(s[1]) - if err != nil { - common.LogWarn(fmt.Sprintf("Invalid count on line %d, skipping", i)) - continue - } - - scale[processType] = count - } - - if len(scale) == 0 { - common.LogWarn("No valid entries found in scale file, defaulting to web=1") - scale["web"] = 1 - } - - return scale, nil -} - func getFormations(appName string) (FormationSlice, error) { - scale := FormationSlice{} - data, err := readScaleFile(appName) + formations := FormationSlice{} + processTuples, err := common.PropertyListGet("ps", appName, "scale") if err != nil { - return scale, err + return formations, err } - for processType, quantity := range data { - f := Formation{ - ProcessType: processType, - Quantity: quantity, - } - scale = append(scale, &f) - } - - return scale, nil + return parseProcessTuples(processTuples) } func removeProcfile(appName string) error { @@ -382,13 +265,12 @@ func scaleReport(appName string) error { } func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error { - scale, err := parseProcessTuples(processTuples) + formations, err := parseProcessTuples(processTuples) if err != nil { return err } - common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) - if err := updateScalefile(appName, clearExisting, scale); err != nil { + if err := updateScale(appName, clearExisting, formations); err != nil { return err } @@ -408,17 +290,15 @@ func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) } -func updateScalefile(appName string, clearExisting bool, scaleUpdates map[string]int) error { - if !hasScaleFile(appName) { - if err := generateScalefile(appName); err != nil { +func updateScale(appName string, clearExisting bool, formationUpdates FormationSlice) error { + formations := FormationSlice{} + if !clearExisting { + processTuples, err := common.PropertyListGet("ps", appName, "scale") + if err != nil { return err } - } - scale := make(map[string]int) - if !clearExisting { - var err error - scale, err = readScaleFile(appName) + formations, err = parseProcessTuples(processTuples) if err != nil { return err } @@ -435,30 +315,43 @@ func updateScalefile(appName string, clearExisting bool, scaleUpdates map[string } } - for processType, count := range scaleUpdates { - if procfileExists && !validProcessTypes[processType] && count != 0 { - return fmt.Errorf("%s is not a valid process name to scale up", processType) + foundProcessTypes := map[string]bool{} + updatedFormation := FormationSlice{} + for _, formation := range formationUpdates { + if procfileExists && !validProcessTypes[formation.ProcessType] && formation.Quantity != 0 { + return fmt.Errorf("%s is not a valid process name to scale up", formation.ProcessType) } - scale[processType] = count + + foundProcessTypes[formation.ProcessType] = true + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: formation.ProcessType, + Quantity: formation.Quantity, + }) + } + + for _, formation := range formations { + foundProcessTypes[formation.ProcessType] = true + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: formation.ProcessType, + Quantity: formation.Quantity, + }) } for processType := range validProcessTypes { - count, ok := scale[processType] - if !ok { - count = 0 + if foundProcessTypes[processType] { + continue } - scale[processType] = count + + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: processType, + Quantity: 0, + }) } - content := []string{} - for processType, count := range scale { - content = append(content, fmt.Sprintf("%s=%d", processType, count)) + values := []string{} + for _, formation := range formations { + values = append(values, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) } - scalefilePath := getScalefilePath(appName) - if err := common.WriteSliceToFile(scalefilePath, content); err != nil { - return err - } - - return nil + return common.PropertyListWrite("ps", appName, "scale", values) } diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 2fb208ba5..1ba4703d2 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" dockeroptions "github.com/dokku/dokku/plugins/docker-options" @@ -120,24 +121,15 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error return err } - procfilePath := getProcfilePath(appName) - if !hasScaleFile(appName) || common.FileExists(procfilePath) { - update := func() error { - return updateScalefile(appName, false, make(map[string]int)) - } - if err := common.SuppressOutput(update); err != nil { - return err - } - } - if len(processTuples) == 0 { return scaleReport(appName) } if !canScaleApp(appName) { - return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) + return fmt.Errorf("App %s contains an app.json file with a formations key and cannot be manually scaled", appName) } + common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) return scaleSet(appName, skipDeploy, false, processTuples) } diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 9de576f28..fbd7ae922 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -91,6 +91,29 @@ func TriggerInstall() error { } } + for _, appName := range apps { + dokkuScaleFile := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE") + if common.FileExists(dokkuScaleFile) { + processTuples, err := common.FileToSlice(dokkuScaleFile) + if err != nil { + return err + } + + if err := scaleSet(appName, true, false, processTuples); err != nil { + return err + } + + os.Remove(dokkuScaleFile) + } + + dokkuScaleExtracted := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") + if common.FileExists(dokkuScaleExtracted) { + if err := common.PropertyWrite("ps", appName, "can-scale", strconv.FormatBool(false)); err != nil { + return err + } + } + } + return nil } @@ -151,7 +174,8 @@ func TriggerPostCreate(appName string) error { return err } - return updateScalefile(appName, false, make(map[string]int)) + formations := FormationSlice{} + return updateScale(appName, false, formations) } // TriggerPostDelete destroys the ps properties for a given app container @@ -192,7 +216,7 @@ func TriggerPostStop(appName string) error { }) } -// TriggerPreDeploy ensures an app has an up to date scale file +// TriggerPreDeploy ensures an app has an up to date scale parameters func TriggerPreDeploy(appName string, imageTag string) error { image, err := common.GetDeployingAppImageName(appName, imageTag, "") if err != nil { @@ -207,7 +231,8 @@ func TriggerPreDeploy(appName string, imageTag string) error { return err } - if err := extractOrGenerateScalefile(appName, image); err != nil { + if err := updateScale(appName, false, FormationSlice{}); err != nil { + common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) return err } diff --git a/tests/unit/ps-general.bats b/tests/unit/ps-general.bats index cd237b151..7c8a5bc34 100644 --- a/tests/unit/ps-general.bats +++ b/tests/unit/ps-general.bats @@ -55,7 +55,7 @@ EOF assert_output "node worker.js" } -@test "(ps:scale) update DOKKU_SCALE from Procfile" { +@test "(ps:scale) update formations from Procfile" { local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM From 823629d33207fa65cb70838715f9632060a90c90 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 17:53:25 -0400 Subject: [PATCH 28/37] docs: document ps-can-scale --- docs/development/plugin-triggers.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 95a552aa1..3d89f2ce8 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1800,6 +1800,20 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `ps-can-scale` + +- Description: Sets whether or not a user can scale an app with `ps:scale` +- Invoked by: +- Arguments: `$APP` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` ### `ps-current-scale` - Description: Prints out the current scale contents (process-type=quantity) delimited by newlines. From 7502cbd13d4a862a2b9fbb7ceac4bcc75b8ce09d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 17:53:34 -0400 Subject: [PATCH 29/37] fix: actually build triggers --- plugins/ps/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/Makefile b/plugins/ps/Makefile index e2a68b39a..c8610988d 100644 --- a/plugins/ps/Makefile +++ b/plugins/ps/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/inspect subcommands/rebuild subcommands/report subcommands/restart subcommands/restore subcommands/retire subcommands/scale subcommands/set subcommands/start subcommands/stop -TRIGGERS = triggers/app-restart triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-extract triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/ps-current-scale triggers/report +TRIGGERS = triggers/app-restart triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-extract triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/ps-can-scale triggers/ps-current-scale triggers/ps-set-scale triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = ps From a7593db0c3cb4c2b522ff18228d923fe6eee266d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 17:53:57 -0400 Subject: [PATCH 30/37] refactor: allow specifying clearExisting when calling ps-set-scale --- docs/development/plugin-triggers.md | 2 +- plugins/app-json/functions.go | 4 +++- plugins/ps/src/triggers/triggers.go | 3 ++- plugins/ps/triggers.go | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 3d89f2ce8..1181a9cff 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1833,7 +1833,7 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x - Description: Sets the scale for an app based on a specified formation (process-type=quantity). Any unspecified process types will be left as is. - Invoked by: -- Arguments: `$APP $SKIP_DEPLOY [$PROCESS_TUPLE...]` +- Arguments: `$APP $SKIP_DEPLOY $CLEAR_EXISTING [$PROCESS_TUPLE...]` - Example: ```shell diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 7ca481fc4..41f671295 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -496,7 +496,9 @@ func setScale(appName string, image string) error { return err } - args := []string{appName, "true"} + skipDeploy := true + clearExisting := false + args := []string{appName, strconv.FormatBool(skipDeploy), strconv.FormatBool(clearExisting)} for processType, formation := range appJSON.Formation { args = append(args, fmt.Sprintf("%s=%d", processType, formation.Quantity)) } diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index bad436b7b..b1581f446 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -86,7 +86,8 @@ func main() { appName := flag.Arg(0) appName, args := common.ShiftString(flag.Args()) skipDeploy, processTuples := common.ShiftString(args) - err = ps.TriggerPsSetScale(appName, common.ToBool(skipDeploy), processTuples) + clearExisting, processTuples := common.ShiftString(args) + err = ps.TriggerPsSetScale(appName, common.ToBool(skipDeploy), common.ToBool(clearExisting), processTuples) case "report": appName := flag.Arg(0) err = ps.ReportSingleApp(appName, "", "") diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index fbd7ae922..49b631f1d 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -315,6 +315,6 @@ func TriggerPsCurrentScale(appName string) error { } // TriggerPsSetScale configures the scale parameters for a given app -func TriggerPsSetScale(appName string, skipDeploy bool, processTuples []string) error { - return scaleSet(appName, skipDeploy, true, processTuples) +func TriggerPsSetScale(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error { + return scaleSet(appName, skipDeploy, clearExisting, processTuples) } From eb55ba5dd04706f3c8ac810f666cc10508f54854 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 18:46:41 -0400 Subject: [PATCH 31/37] fix: correct guard --- plugins/app-json/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 41f671295..93688a1ae 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -503,7 +503,7 @@ func setScale(appName string, image string) error { args = append(args, fmt.Sprintf("%s=%d", processType, formation.Quantity)) } - if len(args) == 2 { + if len(args) == 3 { return common.PlugnTrigger("ps-can-scale", []string{appName, "true"}...) } From fcd16e849deaeb38333f323ab2820942fa145c8f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 19:06:25 -0400 Subject: [PATCH 32/37] fix: set correct initial formation --- plugins/ps/triggers.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 49b631f1d..1b3fe9463 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -174,7 +174,12 @@ func TriggerPostCreate(appName string) error { return err } - formations := FormationSlice{} + formations := FormationSlice{ + &Formation{ + ProcessType: "web", + Quantity: 1, + }, + } return updateScale(appName, false, formations) } From 06cb7a0d55619abd7d755c9dbf9880c9b2f776ee Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 19:06:33 -0400 Subject: [PATCH 33/37] fix: properly update formations --- plugins/ps/functions.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 134f97f32..991fdaa64 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -330,6 +330,10 @@ func updateScale(appName string, clearExisting bool, formationUpdates FormationS } for _, formation := range formations { + if foundProcessTypes[formation.ProcessType] { + continue + } + foundProcessTypes[formation.ProcessType] = true updatedFormation = append(updatedFormation, &Formation{ ProcessType: formation.ProcessType, @@ -349,7 +353,7 @@ func updateScale(appName string, clearExisting bool, formationUpdates FormationS } values := []string{} - for _, formation := range formations { + for _, formation := range updatedFormation { values = append(values, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) } From 4967cc1d1c68f2de6f4d8c4b52a989b9c0cb72a9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 20:13:50 -0400 Subject: [PATCH 34/37] fix: do not manually create an app during tests This will cause weird things to happen due to not initializing the app correctly. --- tests/unit/nginx-vhosts_2.bats | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/nginx-vhosts_2.bats b/tests/unit/nginx-vhosts_2.bats index ed985806f..7769a058d 100644 --- a/tests/unit/nginx-vhosts_2.bats +++ b/tests/unit/nginx-vhosts_2.bats @@ -36,9 +36,14 @@ teardown() { assert_ssl_domain "wildcard2.dokku.me" } -@test "(nginx-vhosts) nginx:build-config (wildcard SSL and unrelated domain)" { +@test "(nginx-vhosts) nginx:build-config (wildcard SSL and unrelated domain) 1" { destroy_app TEST_APP="${TEST_APP}.example.com" + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + setup_test_tls wildcard deploy_app nodejs-express dokku@dokku.me:$TEST_APP run /bin/bash -c "dokku nginx:show-config $TEST_APP | grep -e '*.dokku.me' | wc -l" From 0decb386d531d799e949dabd4c06da30b3d6464e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 21:52:05 -0400 Subject: [PATCH 35/37] fix: correct issues in parsing and writing an updated app.json --- plugins/app-json/functions.go | 6 +++--- plugins/ps/src/triggers/triggers.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index 93688a1ae..d6884f8e6 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -561,13 +561,13 @@ func injectDokkuScale(appName string, image string) error { } processType := procParts[0] - procCount, err := strconv.Atoi(procParts[1]) + quantity, err := strconv.Atoi(procParts[1]) if err != nil { continue } appJSON.Formation[processType] = Formation{ - Quantity: procCount, + Quantity: quantity, } } @@ -576,5 +576,5 @@ func injectDokkuScale(appName string, image string) error { return err } - return ioutil.WriteFile(dokkuScaleFile, b, 0644) + return ioutil.WriteFile(GetAppjsonPath(appName), b, 0644) } diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index b1581f446..9593c7d6e 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -85,7 +85,7 @@ func main() { case "ps-set-scale": appName := flag.Arg(0) appName, args := common.ShiftString(flag.Args()) - skipDeploy, processTuples := common.ShiftString(args) + skipDeploy, args := common.ShiftString(args) clearExisting, processTuples := common.ShiftString(args) err = ps.TriggerPsSetScale(appName, common.ToBool(skipDeploy), common.ToBool(clearExisting), processTuples) case "report": From 7058989941b8e89b5a432eda9b331cc85429aeea Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 22:34:42 -0400 Subject: [PATCH 36/37] chore: delete this file on import --- plugins/ps/triggers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 1b3fe9463..4723ee9af 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -111,6 +111,7 @@ func TriggerInstall() error { if err := common.PropertyWrite("ps", appName, "can-scale", strconv.FormatBool(false)); err != nil { return err } + os.Remove(dokkuScaleExtracted) } } From a8a43812a041df941ea78b73c873e8e27e24cd7a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 23:15:01 -0400 Subject: [PATCH 37/37] chore: make deprecation notice more readable --- plugins/app-json/functions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index d6884f8e6..1a3c6dd7f 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -543,10 +543,10 @@ func injectDokkuScale(appName string, image string) error { } if appJSON.Formation == nil { - common.LogWarn("Deprecated: Use the 'formation' key in app.json to specify scaling instead of DOKKU_SCALE") + common.LogWarn("Deprecated: Injecting scale settings from DOKKU_SCALE file. Use the 'formation' key the app.json file to specify scaling instead of 'DOKKU_SCALE'.") appJSON.Formation = make(map[string]Formation) } else { - common.LogWarn("Deprecated: DOKKU_SCALE ignored in favor of 'formation' key in app.json") + common.LogWarn("Deprecated: DOKKU_SCALE ignored in favor of 'formation' key in the app.json") return nil }