From 236a2797934fb0fc381ccdd6b0a143ca0b611daa Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 15:58:20 -0400 Subject: [PATCH 01/67] chore: add todos --- plugins/scheduler-docker-local/scheduler-deploy | 1 + plugins/scheduler-docker-local/scheduler-run | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index 6f4c7c0f5..a987d7f66 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -215,6 +215,7 @@ fn-scheduler-docker-local-extract-start-cmd() { return fi + # TODO: remove direct procfile access DOKKU_DOCKERFILE_START_CMD=$(config_get "$APP" DOKKU_DOCKERFILE_START_CMD || true) DOKKU_PROCFILE_START_CMD=$(get_cmd_from_procfile "$APP" "$PROC_TYPE" "$PORT") START_CMD=${DOKKU_DOCKERFILE_START_CMD:-$DOKKU_PROCFILE_START_CMD} diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index 074b90c33..fe2e2f299 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -63,6 +63,7 @@ trigger-scheduler-docker-local-scheduler-run() { [[ "$IMAGE_SOURCE_TYPE" == "herokuish" ]] && local EXEC_CMD="/exec" + # TODO: remove direct procfile access DOKKU_QUIET_OUTPUT=1 extract_procfile "$APP" POTENTIAL_PROCFILE_KEY="$1" From ff7429fd00b5849a2676dc783ce26e83b8d1349f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 16:00:55 -0400 Subject: [PATCH 02/67] feat: upgrade go-procfile-util We should ensure we are usinng a new version of it to enable better handling of errors in the Procfile. --- debian/control | 2 +- docs/appendices/0.22.0-migration-guide.md | 2 +- rpm.mk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/control b/debian/control index 5af69b58d..6945e953b 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.21.4 Section: web Priority: optional Architecture: amd64 -Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.10.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, net-tools, software-properties-common, procfile-util, python-software-properties | python3-software-properties, rsyslog, dos2unix, jq +Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.10.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, net-tools, software-properties-common, procfile-util (>= 0.9.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq Recommends: herokuish (>= 0.3.4), parallel, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/docs/appendices/0.22.0-migration-guide.md b/docs/appendices/0.22.0-migration-guide.md index f1ff2c203..55f04a508 100644 --- a/docs/appendices/0.22.0-migration-guide.md +++ b/docs/appendices/0.22.0-migration-guide.md @@ -3,5 +3,5 @@ ## Changes - Underscores are no longer valid characters in app names. Please rename applications before upgrading. -- Process type names specified in Procfile may no longer use characters not valid in DNS records. +- Process type names specified in Procfile may no longer use characters not valid in DNS Label Names ([RFC 1123](https://tools.ietf.org/html/rfc1123)). - The minimum Docker version is now 17.05.0. diff --git a/rpm.mk b/rpm.mk index 15fe18a04..feab42f8b 100644 --- a/rpm.mk +++ b/rpm.mk @@ -42,7 +42,7 @@ endif --depends 'nc' \ --depends 'nginx >= 1.8.0' \ --depends 'plugn' \ - --depends 'procfile-util' \ + --depends 'procfile-util >= 0.9.0' \ --depends '/usr/bin/python3' \ --depends 'sshcommand >= 0.10.0' \ --depends 'sudo' \ From 943e032d127bebade3840a8fdee178df19a6f54c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 17:24:31 -0400 Subject: [PATCH 03/67] refactor: use path/filepath instead of path for joining path parts --- plugins/buildpacks/triggers.go | 4 ++-- plugins/common/properties.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/buildpacks/triggers.go b/plugins/buildpacks/triggers.go index 26481c634..3f0c0f528 100644 --- a/plugins/buildpacks/triggers.go +++ b/plugins/buildpacks/triggers.go @@ -4,7 +4,7 @@ import ( "bufio" "fmt" "os" - "path" + "path/filepath" "github.com/dokku/dokku/plugins/common" ) @@ -58,7 +58,7 @@ func TriggerPostExtract(appName string, sourceWorkDir string) { return } - buildpacksPath := path.Join(sourceWorkDir, ".buildpacks") + buildpacksPath := filepath.Join(sourceWorkDir, ".buildpacks") file, err := os.OpenFile(buildpacksPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600) if err != nil { common.LogFail(fmt.Sprintf("Error writing .buildpacks file: %s", err.Error())) diff --git a/plugins/common/properties.go b/plugins/common/properties.go index 437ac12ce..f164a9c55 100644 --- a/plugins/common/properties.go +++ b/plugins/common/properties.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "os" "os/user" - "path" + "path/filepath" "reflect" "strconv" "strings" @@ -447,17 +447,17 @@ func PropertySetup(pluginName string) error { func getPropertyPath(pluginName string, appName string, property string) string { pluginAppConfigRoot := getPluginAppPropertyPath(pluginName, appName) - return path.Join(pluginAppConfigRoot, property) + return filepath.Join(pluginAppConfigRoot, property) } // getPluginAppPropertyPath returns the plugin property path for a given plugin/app combination func getPluginAppPropertyPath(pluginName string, appName string) string { - return path.Join(getPluginConfigPath(pluginName), appName) + return filepath.Join(getPluginConfigPath(pluginName), appName) } // getPluginConfigPath returns the plugin property path for a given plugin func getPluginConfigPath(pluginName string) string { - return path.Join(MustGetEnv("DOKKU_LIB_ROOT"), "config", pluginName) + return filepath.Join(MustGetEnv("DOKKU_LIB_ROOT"), "config", pluginName) } // makePluginAppPropertyPath ensures that a property path exists From 66696d0b466950b04936984868c3c0c499c7a96a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 17:25:04 -0400 Subject: [PATCH 04/67] refactor: export SetPermissions function --- plugins/common/common.go | 31 +++++++++++++++++++++++ plugins/common/properties.go | 48 ++++++------------------------------ 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index 36cb8760b..8b564b164 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "os/exec" + "os/user" "path/filepath" "reflect" "regexp" @@ -804,6 +805,36 @@ func PlugnTriggerSetup(triggerName string, args ...string) *sh.Session { return sh.Command("plugn", shellArgs...) } +// SetPermissions sets the proper owner and filemode for a given file +func SetPermissions(path string, fileMode os.FileMode) error { + if err := os.Chmod(path, fileMode); err != nil { + return err + } + + systemGroup := GetenvWithDefault("DOKKU_SYSTEM_GROUP", "dokku") + systemUser := GetenvWithDefault("DOKKU_SYSTEM_USER", "dokku") + + group, err := user.LookupGroup(systemGroup) + if err != nil { + return err + } + user, err := user.Lookup(systemUser) + if err != nil { + return err + } + + uid, err := strconv.Atoi(user.Uid) + if err != nil { + return err + } + + gid, err := strconv.Atoi(group.Gid) + if err != nil { + return err + } + return os.Chown(path, uid, gid) +} + func times(str string, n int) (out string) { for i := 0; i < n; i++ { out += str diff --git a/plugins/common/properties.go b/plugins/common/properties.go index f164a9c55..c3b8a748b 100644 --- a/plugins/common/properties.go +++ b/plugins/common/properties.go @@ -6,10 +6,8 @@ import ( "fmt" "io/ioutil" "os" - "os/user" "path/filepath" "reflect" - "strconv" "strings" ) @@ -174,7 +172,7 @@ func PropertyListAdd(pluginName string, appName string, property string, value s } file.Chmod(0600) - setPermissions(propertyPath, 0600) + SetPermissions(propertyPath, 0600) return nil } @@ -301,7 +299,7 @@ func PropertyListRemove(pluginName string, appName string, property string, valu } file.Chmod(0600) - setPermissions(propertyPath, 0600) + SetPermissions(propertyPath, 0600) if !found { return errors.New("Property not found, nothing was removed") @@ -337,7 +335,7 @@ func PropertyListRemoveByPrefix(pluginName string, appName string, property stri } file.Chmod(0600) - setPermissions(propertyPath, 0600) + SetPermissions(propertyPath, 0600) if !found { return errors.New("Property not found, nothing was removed") @@ -390,7 +388,7 @@ func PropertyListSet(pluginName string, appName string, property string, value s } file.Chmod(0600) - setPermissions(propertyPath, 0600) + SetPermissions(propertyPath, 0600) return nil } @@ -429,7 +427,7 @@ func PropertyWrite(pluginName string, appName string, property string, value str fmt.Fprintf(file, value) file.Chmod(0600) - setPermissions(propertyPath, 0600) + SetPermissions(propertyPath, 0600) return nil } @@ -439,10 +437,10 @@ func PropertySetup(pluginName string) error { if err := os.MkdirAll(pluginConfigRoot, 0755); err != nil { return err } - if err := setPermissions(path.Join(MustGetEnv("DOKKU_LIB_ROOT"), "config"), 0755); err != nil { + if err := SetPermissions(filepath.Join(MustGetEnv("DOKKU_LIB_ROOT"), "config"), 0755); err != nil { return err } - return setPermissions(pluginConfigRoot, 0755) + return SetPermissions(pluginConfigRoot, 0755) } func getPropertyPath(pluginName string, appName string, property string) string { @@ -466,35 +464,5 @@ func makePluginAppPropertyPath(pluginName string, appName string) error { if err := os.MkdirAll(pluginAppConfigRoot, 0755); err != nil { return err } - return setPermissions(pluginAppConfigRoot, 0755) -} - -// setPermissions sets the proper owner and filemode for a given file -func setPermissions(path string, fileMode os.FileMode) error { - if err := os.Chmod(path, fileMode); err != nil { - return err - } - - systemGroup := GetenvWithDefault("DOKKU_SYSTEM_GROUP", "dokku") - systemUser := GetenvWithDefault("DOKKU_SYSTEM_USER", "dokku") - - group, err := user.LookupGroup(systemGroup) - if err != nil { - return err - } - user, err := user.Lookup(systemUser) - if err != nil { - return err - } - - uid, err := strconv.Atoi(user.Uid) - if err != nil { - return err - } - - gid, err := strconv.Atoi(group.Gid) - if err != nil { - return err - } - return os.Chown(path, uid, gid) + return SetPermissions(pluginAppConfigRoot, 0755) } From bd132ed7c61121f33235e6c1a3ad868245519f3f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 17:25:25 -0400 Subject: [PATCH 05/67] feat: implement helper functions for dockeroptions --- plugins/docker-options/dockeroptions.go | 102 ++++++++++++++++++++++++ plugins/docker-options/go.mod | 12 +++ plugins/docker-options/go.sum | 8 ++ 3 files changed, 122 insertions(+) create mode 100644 plugins/docker-options/dockeroptions.go create mode 100644 plugins/docker-options/go.mod create mode 100644 plugins/docker-options/go.sum diff --git a/plugins/docker-options/dockeroptions.go b/plugins/docker-options/dockeroptions.go new file mode 100644 index 000000000..63fce5284 --- /dev/null +++ b/plugins/docker-options/dockeroptions.go @@ -0,0 +1,102 @@ +package dockeroptions + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +// AddDockerOptionToPhases adds docker option to specified phases +func AddDockerOptionToPhases(appName string, phases []string, option string) error { + for _, phase := range phases { + if err := touchPhaseFile(appName, phase); err != nil { + return err + } + + options, err := GetDockerOptionsForPhase(appName, phase) + if err != nil { + return err + } + + options = append(options, option) + sort.Strings(options) + if err = writeDockerOptionsForPhase(appName, phase, options); err != nil { + return err + } + } + return nil +} + +// GetDockerOptionsForPhase returns the docker options for the specified phase +func GetDockerOptionsForPhase(appName string, phase string) ([]string, error) { + options := []string{} + + if err := touchPhaseFile(appName, phase); err != nil { + return options, err + } + + phaseFilePath := getPhaseFilePath(appName, phase) + file, err := os.Open(phaseFilePath) + if err != nil { + return options, fmt.Errorf("Unable to open docker options phase file %s.%s: %s", appName, appName, err.Error()) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + options = append(options, strings.TrimSpace(scanner.Text())) + } + + if err = scanner.Err(); err != nil { + return options, fmt.Errorf("Unable to read docker options phase file %s.%s: %s", appName, appName, err.Error()) + } + + return options, nil +} + +func getPhaseFilePath(appName string, phase string) string { + return filepath.Join(common.MustGetEnv("DOKKU_ROOT"), appName, "DOCKER_OPTIONS_"+strings.ToUpper(phase)) +} + +func touchPhaseFile(appName string, phase string) error { + phaseFilePath := getPhaseFilePath(appName, phase) + + _, err := os.Stat(phaseFilePath) + if !os.IsNotExist(err) { + return nil + } + + file, err := os.Create(phaseFilePath) + if err != nil { + return fmt.Errorf("Unable to create docker options phase file %s.%s: %s", appName, phase, err.Error()) + } + defer file.Close() + + return nil +} + +func writeDockerOptionsForPhase(appName string, phase string, options []string) error { + phaseFilePath := getPhaseFilePath(appName, phase) + file, err := os.OpenFile(phaseFilePath, os.O_RDWR|os.O_TRUNC, 0600) + if err != nil { + return err + } + + w := bufio.NewWriter(file) + for _, option := range options { + fmt.Fprintln(w, option) + } + if err = w.Flush(); err != nil { + return fmt.Errorf("Unable to update docker options phase file %s.%s: %s", appName, phase, err.Error()) + } + + file.Chmod(0600) + common.SetPermissions(phaseFilePath, 0600) + + return nil +} diff --git a/plugins/docker-options/go.mod b/plugins/docker-options/go.mod new file mode 100644 index 000000000..7d47c0fa0 --- /dev/null +++ b/plugins/docker-options/go.mod @@ -0,0 +1,12 @@ +module github.com/dokku/dokku/plugins/docker-options + +go 1.14 + +require ( + github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect + github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 + github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect +) + +replace github.com/dokku/dokku/plugins/common => ../common diff --git a/plugins/docker-options/go.sum b/plugins/docker-options/go.sum new file mode 100644 index 000000000..ee7942d28 --- /dev/null +++ b/plugins/docker-options/go.sum @@ -0,0 +1,8 @@ +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= From 1b4559c4a468bb075575381069393165e6328e7b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 17:26:48 -0400 Subject: [PATCH 06/67] chore: drop unused ps import --- plugins/nginx-vhosts/functions | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index 1f2831d49..1aeabc093 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -4,7 +4,6 @@ set -eo pipefail source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/certs/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" source "$PLUGIN_AVAILABLE_PATH/nginx-vhosts/internal-functions" fn-nginx-log-root() { From 696df4530696caa0aba93e1e51f03ac4f6a72c6f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 17:32:21 -0400 Subject: [PATCH 07/67] refactor: extract release_and_deploy calls to a plugin trigger Better would be to separate the release and deploy aspects into distinct triggers - as well as have it release a specific image - but this is a good first pass. --- docs/development/plugin-triggers.md | 16 ++++++++++++++++ plugins/20_events/release-and-deploy | 1 + plugins/common/functions | 2 +- plugins/common/release-and-deploy | 6 ++++++ plugins/ps/functions | 6 +++--- plugins/tags/subcommands/deploy | 2 +- 6 files changed, 28 insertions(+), 5 deletions(-) create mode 120000 plugins/20_events/release-and-deploy create mode 100755 plugins/common/release-and-deploy diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 2a9f42e67..92c356654 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1654,6 +1654,22 @@ fi plugn trigger receive-app $APP $newrev ``` +### `release-and-deploy` + +- Description: Triggers a release of the image tag and a subsequent deploy +- Invoked by: +- Arguments: `$APP $IMAGE_TAG` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +APP="$1"; + +# TODO +``` + ### `report` - Description: Allows you to report on any custom configuration in use by your application diff --git a/plugins/20_events/release-and-deploy b/plugins/20_events/release-and-deploy new file mode 120000 index 000000000..5178a749f --- /dev/null +++ b/plugins/20_events/release-and-deploy @@ -0,0 +1 @@ +hook \ No newline at end of file diff --git a/plugins/common/functions b/plugins/common/functions index 8f882de11..08908cd6f 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -634,7 +634,7 @@ dokku_receive() { dokku_log_info1 "Building $APP from $IMAGE_SOURCE_TYPE..." DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$APP" DOKKU_APP_TYPE="$IMAGE_SOURCE_TYPE" dokku_build "$APP" "$IMAGE_SOURCE_TYPE" "$TMP_WORK_DIR" - release_and_deploy "$APP" + plugn trigger release-and-deploy "$APP" } docker_cleanup() { diff --git a/plugins/common/release-and-deploy b/plugins/common/release-and-deploy new file mode 100755 index 000000000..26610e28e --- /dev/null +++ b/plugins/common/release-and-deploy @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +release_and_deploy "$@" diff --git a/plugins/ps/functions b/plugins/ps/functions index e1dda2a68..bec1dbfbb 100755 --- a/plugins/ps/functions +++ b/plugins/ps/functions @@ -158,7 +158,7 @@ ps_start() { fi if [[ "$RUNNING" != "true" ]]; then - release_and_deploy "$APP" "$IMAGE_TAG" + plugn trigger release-and-deploy "$APP" "$IMAGE_TAG" else dokku_log_warn "App $APP already running" fi @@ -196,7 +196,7 @@ ps_restart() { local IMAGE_TAG=$(get_running_image_tag "$APP") if (is_deployed "$APP"); then - release_and_deploy "$APP" "$IMAGE_TAG" + plugn trigger release-and-deploy "$APP" "$IMAGE_TAG" else dokku_log_warn "App $APP has not been deployed" fi @@ -235,7 +235,7 @@ ps_scale() { else if [[ "$(fn-ps-can-scale "$APP")" == "true" ]]; then set_scale "$APP" "$@" - release_and_deploy "$APP" "$IMAGE_TAG" + plugn trigger release-and-deploy "$APP" "$IMAGE_TAG" else dokku_log_fail "App contains DOKKU_SCALE file and cannot be manually scaled" fi diff --git a/plugins/tags/subcommands/deploy b/plugins/tags/subcommands/deploy index 227b0f30a..e78e86202 100755 --- a/plugins/tags/subcommands/deploy +++ b/plugins/tags/subcommands/deploy @@ -14,7 +14,7 @@ cmd-tags-deploy() { local IMAGE="$(get_app_image_name "$APP" "$IMAGE_TAG")" verify_image "$IMAGE" || return 1 - release_and_deploy "$APP" "$IMAGE_TAG" + plugn trigger release-and-deploy "$APP" "$IMAGE_TAG" if [[ "$IMAGE_TAG" != "latest" ]]; then local DOKKU_SCHEDULER="$(get_app_scheduler "$APP")" From 33d37220606ff1b40f7c22ac3c4159f18f953b8b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 19:15:55 -0400 Subject: [PATCH 08/67] refactor: move common io-related functions into new file --- plugins/common/common.go | 100 ------------- plugins/common/common_test.go | 34 +---- plugins/common/io.go | 191 ++++++++++++++++++++++++ plugins/common/io_test.go | 28 ++++ plugins/docker-options/dockeroptions.go | 18 +-- 5 files changed, 224 insertions(+), 147 deletions(-) create mode 100644 plugins/common/io.go create mode 100644 plugins/common/io_test.go diff --git a/plugins/common/common.go b/plugins/common/common.go index 8b564b164..d90d9162d 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -1,14 +1,12 @@ package common import ( - "bufio" "bytes" "errors" "fmt" "io/ioutil" "os" "os/exec" - "os/user" "path/filepath" "reflect" "regexp" @@ -439,16 +437,6 @@ func ContainerIsRunning(containerID string) bool { return strings.TrimSpace(string(b[:])) == "true" } -// DirectoryExists returns if a path exists and is a directory -func DirectoryExists(filePath string) bool { - fi, err := os.Stat(filePath) - if err != nil { - return false - } - - return fi.IsDir() -} - // DockerContainerCreate creates a new container and returns the container ID func DockerContainerCreate(image string, containerCreateArgs []string) (string, error) { cmd := []string{ @@ -513,36 +501,6 @@ func DokkuApps() (apps []string, err error) { return } -// FileToSlice reads in all the lines from a file into a string slice -func FileToSlice(filePath string) (lines []string, err error) { - f, err := os.Open(filePath) - if err != nil { - return - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - text := strings.TrimSpace(scanner.Text()) - if text == "" { - continue - } - lines = append(lines, text) - } - err = scanner.Err() - return -} - -// FileExists returns if a path exists and is a file -func FileExists(filePath string) bool { - fi, err := os.Stat(filePath) - if err != nil { - return false - } - - return fi.Mode().IsRegular() -} - // GetAppImageName returns image identifier for a given app, tag tuple. validate if tag is presented func GetAppImageName(appName, imageTag, imageRepo string) (imageName string) { err := VerifyAppName(appName) @@ -565,11 +523,6 @@ func GetAppImageName(appName, imageTag, imageRepo string) (imageName string) { return } -// IsAbsPath returns 0 if input path is absolute -func IsAbsPath(path string) bool { - return strings.HasPrefix(path, "/") -} - // IsDeployed returns true if given app has a running container func IsDeployed(appName string) bool { files, err := ioutil.ReadDir(AppRoot(appName)) @@ -612,29 +565,6 @@ func GetenvWithDefault(key string, defaultValue string) (val string) { return } -// ReadFirstLine gets the first line of a file that has contents and returns it -// if there are no contents, an empty string is returned -// will also return an empty string if the file does not exist -func ReadFirstLine(filename string) (text string) { - if !FileExists(filename) { - return - } - f, err := os.Open(filename) - if err != nil { - return - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - if text = strings.TrimSpace(scanner.Text()); text == "" { - continue - } - return - } - return -} - // ReportSingleApp is an internal function that displays a report for an app func ReportSingleApp(reportType string, appName string, infoFlag string, infoFlags map[string]string, trimPrefix bool, uppercaseFirstCharacter bool) error { flags := []string{} @@ -805,36 +735,6 @@ func PlugnTriggerSetup(triggerName string, args ...string) *sh.Session { return sh.Command("plugn", shellArgs...) } -// SetPermissions sets the proper owner and filemode for a given file -func SetPermissions(path string, fileMode os.FileMode) error { - if err := os.Chmod(path, fileMode); err != nil { - return err - } - - systemGroup := GetenvWithDefault("DOKKU_SYSTEM_GROUP", "dokku") - systemUser := GetenvWithDefault("DOKKU_SYSTEM_USER", "dokku") - - group, err := user.LookupGroup(systemGroup) - if err != nil { - return err - } - user, err := user.Lookup(systemUser) - if err != nil { - return err - } - - uid, err := strconv.Atoi(user.Uid) - if err != nil { - return err - } - - gid, err := strconv.Atoi(group.Gid) - if err != nil { - return err - } - return os.Chown(path, uid, gid) -} - func times(str string, n int) (out string) { for i := 0; i < n; i++ { out += str diff --git a/plugins/common/common_test.go b/plugins/common/common_test.go index fedd2b1eb..4fb32ea84 100644 --- a/plugins/common/common_test.go +++ b/plugins/common/common_test.go @@ -10,10 +10,10 @@ import ( ) var ( - testAppName = "test-app-1" - testAppDir = strings.Join([]string{"/home/dokku/", testAppName}, "") - testEnvFile = strings.Join([]string{testAppDir, "/ENV"}, "") - testEnvLine = "export testKey=TESTING" + testAppName = "test-app-1" + testAppDir = strings.Join([]string{"/home/dokku/", testAppName}, "") + testEnvFile = strings.Join([]string{testAppDir, "/ENV"}, "") + testEnvLine = "export testKey=TESTING" testAppName2 = "01-test-app-1" testAppDir2 = strings.Join([]string{"/home/dokku/", testAppName2}, "") testEnvFile2 = strings.Join([]string{testAppDir2, "/ENV"}, "") @@ -95,32 +95,6 @@ func TestCommonDokkuApps(t *testing.T) { teardownTestApp() } -func TestCommonFileToSlice(t *testing.T) { - RegisterTestingT(t) - Expect(setupTestApp()).To(Succeed()) - lines, err := FileToSlice(testEnvFile) - Expect(err).NotTo(HaveOccurred()) - Expect(lines).To(Equal([]string{testEnvLine})) - teardownTestApp() -} - -func TestCommonFileExists(t *testing.T) { - RegisterTestingT(t) - Expect(setupTestApp()).To(Succeed()) - Expect(FileExists(testEnvFile)).To(BeTrue()) - teardownTestApp() -} - -func TestCommonReadFirstLine(t *testing.T) { - RegisterTestingT(t) - line := ReadFirstLine(testEnvFile) - Expect(line).To(Equal("")) - Expect(setupTestApp()).To(Succeed()) - line = ReadFirstLine(testEnvFile) - Expect(line).To(Equal(testEnvLine)) - teardownTestApp() -} - func TestCommonStripInlineComments(t *testing.T) { RegisterTestingT(t) text := StripInlineComments(strings.Join([]string{testEnvLine, "# testing comment"}, " ")) diff --git a/plugins/common/io.go b/plugins/common/io.go new file mode 100644 index 000000000..f57b20e42 --- /dev/null +++ b/plugins/common/io.go @@ -0,0 +1,191 @@ +package common + +import ( + "bufio" + "fmt" + "io" + "os" + "os/user" + "strconv" + "strings" +) + +// CopyFile copies a file from src to dst. If src and dst files exist, and are +// the same, then return success. Otherise, attempt to create a hard link +// between the two files. If that fail, copy the file contents from src to dst. +// FROM: https://stackoverflow.com/a/21067803/1515875 +func CopyFile(src, dst string) (err error) { + sfi, err := os.Stat(src) + if err != nil { + return + } + if !sfi.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String()) + } + dfi, err := os.Stat(dst) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(dfi.Mode().IsRegular()) { + return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String()) + } + if os.SameFile(sfi, dfi) { + return + } + } + if err = os.Link(src, dst); err == nil { + return + } + err = copyFileContents(src, dst) + return +} + +// copyFileContents copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. +// FROM: https://stackoverflow.com/a/21067803/1515875 +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +// DirectoryExists returns if a path exists and is a directory +func DirectoryExists(filename string) bool { + fi, err := os.Stat(filename) + if err != nil { + return false + } + + return fi.IsDir() +} + +// FileToSlice reads in all the lines from a file into a string slice +func FileToSlice(filename string) (lines []string, err error) { + f, err := os.Open(filename) + if err != nil { + return + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + lines = append(lines, text) + } + err = scanner.Err() + return +} + +// FileExists returns if a path exists and is a file +func FileExists(filename string) bool { + fi, err := os.Stat(filename) + if err != nil { + return false + } + + return fi.Mode().IsRegular() +} + +// IsAbsPath returns 0 if input path is absolute +func IsAbsPath(path string) bool { + return strings.HasPrefix(path, "/") +} + +// ReadFirstLine gets the first line of a file that has contents and returns it +// if there are no contents, an empty string is returned +// will also return an empty string if the file does not exist +func ReadFirstLine(filename string) (text string) { + if !FileExists(filename) { + return + } + f, err := os.Open(filename) + if err != nil { + return + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if text = strings.TrimSpace(scanner.Text()); text == "" { + continue + } + return + } + return +} + +// SetPermissions sets the proper owner and filemode for a given file +func SetPermissions(path string, fileMode os.FileMode) error { + if err := os.Chmod(path, fileMode); err != nil { + return err + } + + systemGroup := GetenvWithDefault("DOKKU_SYSTEM_GROUP", "dokku") + systemUser := GetenvWithDefault("DOKKU_SYSTEM_USER", "dokku") + + group, err := user.LookupGroup(systemGroup) + if err != nil { + return err + } + user, err := user.Lookup(systemUser) + if err != nil { + return err + } + + uid, err := strconv.Atoi(user.Uid) + if err != nil { + return err + } + + gid, err := strconv.Atoi(group.Gid) + if err != nil { + return err + } + return os.Chown(path, uid, gid) +} + +func WriteSliceToFile(filename string, lines []string) error { + file, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC, 0600) + if err != nil { + return err + } + + w := bufio.NewWriter(file) + for _, line := range lines { + fmt.Fprintln(w, line) + } + if err = w.Flush(); err != nil { + return err + } + + file.Chmod(0600) + SetPermissions(filename, 0600) + + return nil +} diff --git a/plugins/common/io_test.go b/plugins/common/io_test.go new file mode 100644 index 000000000..856206a87 --- /dev/null +++ b/plugins/common/io_test.go @@ -0,0 +1,28 @@ + +import "testing" + +func TestCommonFileToSlice(t *testing.T) { + RegisterTestingT(t) + Expect(setupTestApp()).To(Succeed()) + lines, err := FileToSlice(testEnvFile) + Expect(err).NotTo(HaveOccurred()) + Expect(lines).To(Equal([]string{testEnvLine})) + teardownTestApp() +} + +func TestCommonFileExists(t *testing.T) { + RegisterTestingT(t) + Expect(setupTestApp()).To(Succeed()) + Expect(FileExists(testEnvFile)).To(BeTrue()) + teardownTestApp() +} + +func TestCommonReadFirstLine(t *testing.T) { + RegisterTestingT(t) + line := ReadFirstLine(testEnvFile) + Expect(line).To(Equal("")) + Expect(setupTestApp()).To(Succeed()) + line = ReadFirstLine(testEnvFile) + Expect(line).To(Equal(testEnvLine)) + teardownTestApp() +} diff --git a/plugins/docker-options/dockeroptions.go b/plugins/docker-options/dockeroptions.go index 63fce5284..a1d44e6b3 100644 --- a/plugins/docker-options/dockeroptions.go +++ b/plugins/docker-options/dockeroptions.go @@ -82,21 +82,5 @@ func touchPhaseFile(appName string, phase string) error { func writeDockerOptionsForPhase(appName string, phase string, options []string) error { phaseFilePath := getPhaseFilePath(appName, phase) - file, err := os.OpenFile(phaseFilePath, os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return err - } - - w := bufio.NewWriter(file) - for _, option := range options { - fmt.Fprintln(w, option) - } - if err = w.Flush(); err != nil { - return fmt.Errorf("Unable to update docker options phase file %s.%s: %s", appName, phase, err.Error()) - } - - file.Chmod(0600) - common.SetPermissions(phaseFilePath, 0600) - - return nil + return common.WriteSliceToFile(phaseFilePath, options) } From 74f27480fc6ac4751f1b2a53a3ed51ecfa2ab468 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 19:39:40 -0400 Subject: [PATCH 09/67] refactor: move all docker-related functions to new file --- plugins/common/common.go | 345 +------------------------------------- plugins/common/docker.go | 350 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+), 342 deletions(-) create mode 100644 plugins/common/docker.go diff --git a/plugins/common/common.go b/plugins/common/common.go index d90d9162d..61197e621 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -1,7 +1,6 @@ package common import ( - "bytes" "errors" "fmt" "io/ioutil" @@ -127,163 +126,6 @@ func CommandUsage(helpHeader string, helpContent string) { fmt.Println(columnize.Format(content, config)) } -// CopyFromImage copies a file from named image to destination -func CopyFromImage(appName string, image string, source string, destination string) error { - if err := VerifyAppName(appName); err != nil { - return err - } - - if !VerifyImage(image) { - return fmt.Errorf("Invalid docker image for copying content") - } - - workDir := "" - if !IsAbsPath(source) { - if IsImageHerokuishBased(image, appName) { - workDir = "/app" - } else { - workDir, _ = DockerInspect(image, "{{.Config.WorkingDir}}") - } - - if workDir != "" { - source = fmt.Sprintf("%s/%s", workDir, source) - } - } - - tmpFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", MustGetEnv("DOKKU_PID"), "CopyFromImage")) - if err != nil { - return fmt.Errorf("Cannot create temporary file: %v", err) - } - - defer tmpFile.Close() - defer os.Remove(tmpFile.Name()) - - globalRunArgs := MustGetEnv("DOKKU_GLOBAL_RUN_ARGS") - createLabelArgs := []string{"--label", fmt.Sprintf("com.dokku.app-name=%s", appName), globalRunArgs} - containerID, err := DockerContainerCreate(image, createLabelArgs) - if err != nil { - return fmt.Errorf("Unable to create temporary container: %v", err) - } - - // docker cp exits with status 1 when run as non-root user when it tries to chown the file - // after successfully copying the file. Thus, we suppress stderr. - // ref: https://github.com/dotcloud/docker/issues/3986 - containerCopyCmd := NewShellCmd(strings.Join([]string{ - DockerBin(), - "container", - "cp", - fmt.Sprintf("%s:%s", containerID, source), - tmpFile.Name(), - }, " ")) - containerCopyCmd.ShowOutput = false - fileCopied := containerCopyCmd.Execute() - - containerRemoveCmd := NewShellCmd(strings.Join([]string{ - DockerBin(), - "container", - "rm", - "--force", - containerID, - }, " ")) - containerRemoveCmd.ShowOutput = false - containerRemoveCmd.Execute() - - if !fileCopied { - return fmt.Errorf("Unable to copy file %s from image", source) - } - - fi, err := os.Stat(tmpFile.Name()) - if err != nil { - return err - } - - if fi.Size() == 0 { - return fmt.Errorf("Unable to copy file %s from image", source) - } - - // workaround for CHECKS file when owner is root. seems to only happen when running inside docker - dos2unixCmd := NewShellCmd(strings.Join([]string{ - "dos2unix", - "-l", - "-n", - tmpFile.Name(), - destination, - }, " ")) - dos2unixCmd.ShowOutput = false - dos2unixCmd.Execute() - - // add trailing newline for certain places where file parsing depends on it - b, err := sh.Command("tail", "-c1", destination).Output() - if string(b) != "" { - f, err := os.OpenFile(destination, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return err - } - defer f.Close() - if _, err := f.WriteString("\n"); err != nil { - return fmt.Errorf("Unable to append trailing newline to copied file: %v", err) - } - } - - return nil -} - -// DockerCleanup cleans up all exited/dead containers and removes all dangling images -func DockerCleanup(appName string, forceCleanup bool) error { - if !forceCleanup { - skipCleanup := false - if appName != "" { - b, _ := PlugnTriggerOutput("config-get", []string{appName, "DOKKU_SKIP_CLEANUP"}...) - output := string(b[:]) - if output == "true" { - skipCleanup = true - } - } - - if skipCleanup || os.Getenv("DOKKU_SKIP_CLEANUP") == "true" { - LogInfo1("DOKKU_SKIP_CLEANUP set. Skipping dokku cleanup") - return nil - } - } - - LogInfo1("Cleaning up...") - scheduler := GetAppScheduler(appName) - if appName == "--global" { - appName = "" - } - - forceCleanupArg := "false" - if forceCleanup { - forceCleanupArg = "true" - } - - if err := PlugnTrigger("scheduler-docker-cleanup", []string{scheduler, appName, forceCleanupArg}...); err != nil { - return fmt.Errorf("Failure while cleaning up app: %s", err) - } - - // delete all non-running and dead containers - exitedContainerIDs, _ := listContainers("exited", appName) - deadContainerIDs, _ := listContainers("dead", appName) - containerIDs := append(exitedContainerIDs, deadContainerIDs...) - - if len(containerIDs) > 0 { - removeContainers(containerIDs) - } - - // delete dangling images - imageIDs, _ := listDanglingImages(appName) - if len(imageIDs) > 0 { - RemoveImages(imageIDs) - } - - if appName != "" { - // delete unused images - pruneUnusedImages(appName) - } - - return nil -} - // GetAppScheduler fetches the scheduler for a given application func GetAppScheduler(appName string) string { if appName == "--global" { @@ -428,51 +270,6 @@ func GetRunningImageTag(appName string) (string, error) { return "", errors.New("No image tag found") } -// ContainerIsRunning checks to see if a container is running -func ContainerIsRunning(containerID string) bool { - b, err := DockerInspect(containerID, "'{{.State.Running}}'") - if err != nil { - return false - } - return strings.TrimSpace(string(b[:])) == "true" -} - -// DockerContainerCreate creates a new container and returns the container ID -func DockerContainerCreate(image string, containerCreateArgs []string) (string, error) { - cmd := []string{ - DockerBin(), - "container", - "create", - } - - cmd = append(cmd, containerCreateArgs...) - cmd = append(cmd, image) - - var stderr bytes.Buffer - containerCreateCmd := NewShellCmd(strings.Join(cmd, " ")) - containerCreateCmd.ShowOutput = false - containerCreateCmd.Command.Stderr = &stderr - b, err := containerCreateCmd.Output() - if err != nil { - return "", errors.New(strings.TrimSpace(stderr.String())) - } - - return strings.TrimSpace(string(b[:])), nil -} - -// DockerInspect runs an inspect command with a given format against a container or image ID -func DockerInspect(containerOrImageID, format string) (output string, err error) { - b, err := sh.Command(DockerBin(), "inspect", "--format", format, containerOrImageID).Output() - if err != nil { - return "", err - } - output = strings.TrimSpace(string(b[:])) - if strings.HasPrefix(output, "'") && strings.HasSuffix(output, "'") { - output = strings.TrimSuffix(strings.TrimPrefix(output, "'"), "'") - } - return -} - // DokkuApps returns a list of all local apps func DokkuApps() (apps []string, err error) { dokkuRoot := MustGetEnv("DOKKU_ROOT") @@ -538,15 +335,6 @@ func IsDeployed(appName string) bool { return false } -// IsImageHerokuishBased returns true if app image is based on herokuish -func IsImageHerokuishBased(image string, appName string) bool { - output, err := DockerInspect(image, "{{range .Config.Env}}{{if eq . \"USER=herokuishuser\" }}{{println .}}{{end}}{{end}}") - if err != nil { - return false - } - return output != "" -} - // MustGetEnv returns env variable or fails if it's not set func MustGetEnv(key string) (val string) { val = os.Getenv(key) @@ -626,10 +414,10 @@ func ShiftString(a []string) (string, []string) { // StripInlineComments removes bash-style comment from input line func StripInlineComments(text string) string { - bytes := []byte(text) + b := []byte(text) re := regexp.MustCompile("(?s)#.*") - bytes = re.ReplaceAll(bytes, nil) - return strings.TrimSpace(string(bytes)) + b = re.ReplaceAll(b, nil) + return strings.TrimSpace(string(b)) } // SuppressOutput suppresses the output of a function unless there is an error @@ -697,23 +485,6 @@ func VerifyAppName(appName string) error { return nil } -// VerifyImage returns true if docker image exists in local repo -func VerifyImage(image string) bool { - imageCmd := NewShellCmd(strings.Join([]string{DockerBin(), "image", "inspect", image}, " ")) - imageCmd.ShowOutput = false - return imageCmd.Execute() -} - -// DockerBin returns a string which contains a path to the current docker binary -func DockerBin() string { - dockerBin := os.Getenv("DOCKER_BIN") - if dockerBin == "" { - dockerBin = "docker" - } - - return dockerBin -} - // PlugnTrigger fire the given plugn trigger with the given args func PlugnTrigger(triggerName string, args ...string) error { return PlugnTriggerSetup(triggerName, args...).Run() @@ -741,113 +512,3 @@ func times(str string, n int) (out string) { } return } - -func listContainers(status string, appName string) ([]string, error) { - command := []string{ - DockerBin(), - "container", - "list", - "--quiet", - "--all", - "--filter", - fmt.Sprintf("status=%v", status), - "--filter", - fmt.Sprintf("label=%v", os.Getenv("DOKKU_CONTAINER_LABEL")), - } - - if appName != "" { - command = append(command, []string{"--filter", fmt.Sprintf("label=com.dokku.app-name=%v", appName)}...) - } - - var stderr bytes.Buffer - listCmd := NewShellCmd(strings.Join(command, " ")) - listCmd.ShowOutput = false - listCmd.Command.Stderr = &stderr - b, err := listCmd.Output() - - if err != nil { - return []string{}, errors.New(strings.TrimSpace(stderr.String())) - } - - output := strings.Split(strings.TrimSpace(string(b[:])), "\n") - return output, nil -} - -func listDanglingImages(appName string) ([]string, error) { - command := []string{ - DockerBin(), - "image", - "list", - "--quiet", - "--filter", - "dangling=true", - } - - if appName != "" { - command = append(command, []string{"--filter", fmt.Sprintf("label=com.dokku.app-name=%v", appName)}...) - } - - var stderr bytes.Buffer - listCmd := NewShellCmd(strings.Join(command, " ")) - listCmd.ShowOutput = false - listCmd.Command.Stderr = &stderr - b, err := listCmd.Output() - - if err != nil { - return []string{}, errors.New(strings.TrimSpace(stderr.String())) - } - - output := strings.Split(strings.TrimSpace(string(b[:])), "\n") - return output, nil -} - -func removeContainers(containerIDs []string) { - command := []string{ - DockerBin(), - "container", - "rm", - } - - command = append(command, containerIDs...) - - var stderr bytes.Buffer - rmCmd := NewShellCmd(strings.Join(command, " ")) - rmCmd.ShowOutput = false - rmCmd.Command.Stderr = &stderr - rmCmd.Execute() -} - -// RemoveImages removes images by ID -func RemoveImages(imageIDs []string) { - command := []string{ - DockerBin(), - "image", - "rm", - } - - command = append(command, imageIDs...) - - var stderr bytes.Buffer - rmCmd := NewShellCmd(strings.Join(command, " ")) - rmCmd.ShowOutput = false - rmCmd.Command.Stderr = &stderr - rmCmd.Execute() -} - -func pruneUnusedImages(appName string) { - command := []string{ - DockerBin(), - "image", - "prune", - "--all", - "--force", - "--filter", - fmt.Sprintf("label=com.dokku.app-name=%v", appName), - } - - var stderr bytes.Buffer - pruneCmd := NewShellCmd(strings.Join(command, " ")) - pruneCmd.ShowOutput = false - pruneCmd.Command.Stderr = &stderr - pruneCmd.Execute() -} diff --git a/plugins/common/docker.go b/plugins/common/docker.go new file mode 100644 index 000000000..dff5c95c2 --- /dev/null +++ b/plugins/common/docker.go @@ -0,0 +1,350 @@ +package common + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/codeskyblue/go-sh" +) + +// ContainerIsRunning checks to see if a container is running +func ContainerIsRunning(containerID string) bool { + b, err := DockerInspect(containerID, "'{{.State.Running}}'") + if err != nil { + return false + } + return strings.TrimSpace(string(b[:])) == "true" +} + +// CopyFromImage copies a file from named image to destination +func CopyFromImage(appName string, image string, source string, destination string) error { + if err := VerifyAppName(appName); err != nil { + return err + } + + if !VerifyImage(image) { + return fmt.Errorf("Invalid docker image for copying content") + } + + workDir := "" + if !IsAbsPath(source) { + if IsImageHerokuishBased(image, appName) { + workDir = "/app" + } else { + workDir, _ = DockerInspect(image, "{{.Config.WorkingDir}}") + } + + if workDir != "" { + source = fmt.Sprintf("%s/%s", workDir, source) + } + } + + tmpFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", MustGetEnv("DOKKU_PID"), "CopyFromImage")) + if err != nil { + return fmt.Errorf("Cannot create temporary file: %v", err) + } + + defer tmpFile.Close() + defer os.Remove(tmpFile.Name()) + + globalRunArgs := MustGetEnv("DOKKU_GLOBAL_RUN_ARGS") + createLabelArgs := []string{"--label", fmt.Sprintf("com.dokku.app-name=%s", appName), globalRunArgs} + containerID, err := DockerContainerCreate(image, createLabelArgs) + if err != nil { + return fmt.Errorf("Unable to create temporary container: %v", err) + } + + // docker cp exits with status 1 when run as non-root user when it tries to chown the file + // after successfully copying the file. Thus, we suppress stderr. + // ref: https://github.com/dotcloud/docker/issues/3986 + containerCopyCmd := NewShellCmd(strings.Join([]string{ + DockerBin(), + "container", + "cp", + fmt.Sprintf("%s:%s", containerID, source), + tmpFile.Name(), + }, " ")) + containerCopyCmd.ShowOutput = false + fileCopied := containerCopyCmd.Execute() + + containerRemoveCmd := NewShellCmd(strings.Join([]string{ + DockerBin(), + "container", + "rm", + "--force", + containerID, + }, " ")) + containerRemoveCmd.ShowOutput = false + containerRemoveCmd.Execute() + + if !fileCopied { + return fmt.Errorf("Unable to copy file %s from image", source) + } + + fi, err := os.Stat(tmpFile.Name()) + if err != nil { + return err + } + + if fi.Size() == 0 { + return fmt.Errorf("Unable to copy file %s from image", source) + } + + // workaround for CHECKS file when owner is root. seems to only happen when running inside docker + dos2unixCmd := NewShellCmd(strings.Join([]string{ + "dos2unix", + "-l", + "-n", + tmpFile.Name(), + destination, + }, " ")) + dos2unixCmd.ShowOutput = false + dos2unixCmd.Execute() + + // add trailing newline for certain places where file parsing depends on it + b, err := sh.Command("tail", "-c1", destination).Output() + if string(b) != "" { + f, err := os.OpenFile(destination, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + if _, err := f.WriteString("\n"); err != nil { + return fmt.Errorf("Unable to append trailing newline to copied file: %v", err) + } + } + + return nil +} + +// DockerBin returns a string which contains a path to the current docker binary +func DockerBin() string { + dockerBin := os.Getenv("DOCKER_BIN") + if dockerBin == "" { + dockerBin = "docker" + } + + return dockerBin +} + +// DockerCleanup cleans up all exited/dead containers and removes all dangling images +func DockerCleanup(appName string, forceCleanup bool) error { + if !forceCleanup { + skipCleanup := false + if appName != "" { + b, _ := PlugnTriggerOutput("config-get", []string{appName, "DOKKU_SKIP_CLEANUP"}...) + output := string(b[:]) + if output == "true" { + skipCleanup = true + } + } + + if skipCleanup || os.Getenv("DOKKU_SKIP_CLEANUP") == "true" { + LogInfo1("DOKKU_SKIP_CLEANUP set. Skipping dokku cleanup") + return nil + } + } + + LogInfo1("Cleaning up...") + scheduler := GetAppScheduler(appName) + if appName == "--global" { + appName = "" + } + + forceCleanupArg := "false" + if forceCleanup { + forceCleanupArg = "true" + } + + if err := PlugnTrigger("scheduler-docker-cleanup", []string{scheduler, appName, forceCleanupArg}...); err != nil { + return fmt.Errorf("Failure while cleaning up app: %s", err) + } + + // delete all non-running and dead containers + exitedContainerIDs, _ := listContainers("exited", appName) + deadContainerIDs, _ := listContainers("dead", appName) + containerIDs := append(exitedContainerIDs, deadContainerIDs...) + + if len(containerIDs) > 0 { + removeContainers(containerIDs) + } + + // delete dangling images + imageIDs, _ := listDanglingImages(appName) + if len(imageIDs) > 0 { + RemoveImages(imageIDs) + } + + if appName != "" { + // delete unused images + pruneUnusedImages(appName) + } + + return nil +} + +// DockerContainerCreate creates a new container and returns the container ID +func DockerContainerCreate(image string, containerCreateArgs []string) (string, error) { + cmd := []string{ + DockerBin(), + "container", + "create", + } + + cmd = append(cmd, containerCreateArgs...) + cmd = append(cmd, image) + + var stderr bytes.Buffer + containerCreateCmd := NewShellCmd(strings.Join(cmd, " ")) + containerCreateCmd.ShowOutput = false + containerCreateCmd.Command.Stderr = &stderr + b, err := containerCreateCmd.Output() + if err != nil { + return "", errors.New(strings.TrimSpace(stderr.String())) + } + + return strings.TrimSpace(string(b[:])), nil +} + +// DockerInspect runs an inspect command with a given format against a container or image ID +func DockerInspect(containerOrImageID, format string) (output string, err error) { + b, err := sh.Command(DockerBin(), "inspect", "--format", format, containerOrImageID).Output() + if err != nil { + return "", err + } + output = strings.TrimSpace(string(b[:])) + if strings.HasPrefix(output, "'") && strings.HasSuffix(output, "'") { + output = strings.TrimSuffix(strings.TrimPrefix(output, "'"), "'") + } + return +} + +// IsImageHerokuishBased returns true if app image is based on herokuish +func IsImageHerokuishBased(image string, appName string) bool { + output, err := DockerInspect(image, "{{range .Config.Env}}{{if eq . \"USER=herokuishuser\" }}{{println .}}{{end}}{{end}}") + if err != nil { + return false + } + return output != "" +} + +// RemoveImages removes images by ID +func RemoveImages(imageIDs []string) { + command := []string{ + DockerBin(), + "image", + "rm", + } + + command = append(command, imageIDs...) + + var stderr bytes.Buffer + rmCmd := NewShellCmd(strings.Join(command, " ")) + rmCmd.ShowOutput = false + rmCmd.Command.Stderr = &stderr + rmCmd.Execute() +} + +// VerifyImage returns true if docker image exists in local repo +func VerifyImage(image string) bool { + imageCmd := NewShellCmd(strings.Join([]string{DockerBin(), "image", "inspect", image}, " ")) + imageCmd.ShowOutput = false + return imageCmd.Execute() +} + +func listContainers(status string, appName string) ([]string, error) { + command := []string{ + DockerBin(), + "container", + "list", + "--quiet", + "--all", + "--filter", + fmt.Sprintf("status=%v", status), + "--filter", + fmt.Sprintf("label=%v", os.Getenv("DOKKU_CONTAINER_LABEL")), + } + + if appName != "" { + command = append(command, []string{"--filter", fmt.Sprintf("label=com.dokku.app-name=%v", appName)}...) + } + + var stderr bytes.Buffer + listCmd := NewShellCmd(strings.Join(command, " ")) + listCmd.ShowOutput = false + listCmd.Command.Stderr = &stderr + b, err := listCmd.Output() + + if err != nil { + return []string{}, errors.New(strings.TrimSpace(stderr.String())) + } + + output := strings.Split(strings.TrimSpace(string(b[:])), "\n") + return output, nil +} + +func listDanglingImages(appName string) ([]string, error) { + command := []string{ + DockerBin(), + "image", + "list", + "--quiet", + "--filter", + "dangling=true", + } + + if appName != "" { + command = append(command, []string{"--filter", fmt.Sprintf("label=com.dokku.app-name=%v", appName)}...) + } + + var stderr bytes.Buffer + listCmd := NewShellCmd(strings.Join(command, " ")) + listCmd.ShowOutput = false + listCmd.Command.Stderr = &stderr + b, err := listCmd.Output() + + if err != nil { + return []string{}, errors.New(strings.TrimSpace(stderr.String())) + } + + output := strings.Split(strings.TrimSpace(string(b[:])), "\n") + return output, nil +} + +func pruneUnusedImages(appName string) { + command := []string{ + DockerBin(), + "image", + "prune", + "--all", + "--force", + "--filter", + fmt.Sprintf("label=com.dokku.app-name=%v", appName), + } + + var stderr bytes.Buffer + pruneCmd := NewShellCmd(strings.Join(command, " ")) + pruneCmd.ShowOutput = false + pruneCmd.Command.Stderr = &stderr + pruneCmd.Execute() +} + +func removeContainers(containerIDs []string) { + command := []string{ + DockerBin(), + "container", + "rm", + } + + command = append(command, containerIDs...) + + var stderr bytes.Buffer + rmCmd := NewShellCmd(strings.Join(command, " ")) + rmCmd.ShowOutput = false + rmCmd.Command.Stderr = &stderr + rmCmd.Execute() +} From c7becc6cad8f5508244f8cb4abe9388536875246 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 19:41:58 -0400 Subject: [PATCH 10/67] refactor: move all subprocess-related code to new file --- plugins/common/common.go | 86 --------------------------------- plugins/common/subprocess.go | 94 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 86 deletions(-) create mode 100644 plugins/common/subprocess.go diff --git a/plugins/common/common.go b/plugins/common/common.go index 61197e621..fdd53aee0 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "path/filepath" "reflect" "regexp" @@ -13,75 +12,11 @@ import ( "strings" "unicode" - sh "github.com/codeskyblue/go-sh" columnize "github.com/ryanuber/columnize" ) type errfunc func() error -// ShellCmd represents a shell command to be run for dokku -type ShellCmd struct { - Env map[string]string - Command *exec.Cmd - CommandString string - Args []string - ShowOutput bool -} - -// NewShellCmd returns a new ShellCmd struct -func NewShellCmd(command string) *ShellCmd { - items := strings.Split(command, " ") - cmd := items[0] - args := items[1:] - return NewShellCmdWithArgs(cmd, args...) -} - -// NewShellCmdWithArgs returns a new ShellCmd struct -func NewShellCmdWithArgs(cmd string, args ...string) *ShellCmd { - commandString := strings.Join(append([]string{cmd}, args...), " ") - - return &ShellCmd{ - Command: exec.Command(cmd, args...), - CommandString: commandString, - Args: args, - ShowOutput: true, - } -} - -func (sc *ShellCmd) setup() { - env := os.Environ() - for k, v := range sc.Env { - env = append(env, fmt.Sprintf("%s=%s", k, v)) - } - sc.Command.Env = env - if sc.ShowOutput { - sc.Command.Stdout = os.Stdout - sc.Command.Stderr = os.Stderr - } -} - -// Execute is a lightweight wrapper around exec.Command -func (sc *ShellCmd) Execute() bool { - sc.setup() - - if err := sc.Command.Run(); err != nil { - return false - } - return true -} - -// Output is a lightweight wrapper around exec.Command.Output() -func (sc *ShellCmd) Output() ([]byte, error) { - sc.setup() - return sc.Command.Output() -} - -// CombinedOutput is a lightweight wrapper around exec.Command.CombinedOutput() -func (sc *ShellCmd) CombinedOutput() ([]byte, error) { - sc.setup() - return sc.Command.CombinedOutput() -} - // AppRoot returns the app root path func AppRoot(appName string) string { dokkuRoot := MustGetEnv("DOKKU_ROOT") @@ -485,27 +420,6 @@ func VerifyAppName(appName string) error { return nil } -// PlugnTrigger fire the given plugn trigger with the given args -func PlugnTrigger(triggerName string, args ...string) error { - return PlugnTriggerSetup(triggerName, args...).Run() -} - -// PlugnTriggerOutput fire the given plugn trigger with the given args -func PlugnTriggerOutput(triggerName string, args ...string) ([]byte, error) { - return PlugnTriggerSetup(triggerName, args...).Output() -} - -// PlugnTriggerSetup sets up a plugn trigger call -func PlugnTriggerSetup(triggerName string, args ...string) *sh.Session { - shellArgs := make([]interface{}, len(args)+2) - shellArgs[0] = "trigger" - shellArgs[1] = triggerName - for i, arg := range args { - shellArgs[i+2] = arg - } - return sh.Command("plugn", shellArgs...) -} - func times(str string, n int) (out string) { for i := 0; i < n; i++ { out += str diff --git a/plugins/common/subprocess.go b/plugins/common/subprocess.go new file mode 100644 index 000000000..179dc0420 --- /dev/null +++ b/plugins/common/subprocess.go @@ -0,0 +1,94 @@ +package common + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/codeskyblue/go-sh" +) + +// ShellCmd represents a shell command to be run for dokku +type ShellCmd struct { + Env map[string]string + Command *exec.Cmd + CommandString string + Args []string + ShowOutput bool +} + +// NewShellCmd returns a new ShellCmd struct +func NewShellCmd(command string) *ShellCmd { + items := strings.Split(command, " ") + cmd := items[0] + args := items[1:] + return NewShellCmdWithArgs(cmd, args...) +} + +// NewShellCmdWithArgs returns a new ShellCmd struct +func NewShellCmdWithArgs(cmd string, args ...string) *ShellCmd { + commandString := strings.Join(append([]string{cmd}, args...), " ") + + return &ShellCmd{ + Command: exec.Command(cmd, args...), + CommandString: commandString, + Args: args, + ShowOutput: true, + } +} + +func (sc *ShellCmd) setup() { + env := os.Environ() + for k, v := range sc.Env { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + sc.Command.Env = env + if sc.ShowOutput { + sc.Command.Stdout = os.Stdout + sc.Command.Stderr = os.Stderr + } +} + +// Execute is a lightweight wrapper around exec.Command +func (sc *ShellCmd) Execute() bool { + sc.setup() + + if err := sc.Command.Run(); err != nil { + return false + } + return true +} + +// Output is a lightweight wrapper around exec.Command.Output() +func (sc *ShellCmd) Output() ([]byte, error) { + sc.setup() + return sc.Command.Output() +} + +// CombinedOutput is a lightweight wrapper around exec.Command.CombinedOutput() +func (sc *ShellCmd) CombinedOutput() ([]byte, error) { + sc.setup() + return sc.Command.CombinedOutput() +} + +// PlugnTrigger fire the given plugn trigger with the given args +func PlugnTrigger(triggerName string, args ...string) error { + return PlugnTriggerSetup(triggerName, args...).Run() +} + +// PlugnTriggerOutput fire the given plugn trigger with the given args +func PlugnTriggerOutput(triggerName string, args ...string) ([]byte, error) { + return PlugnTriggerSetup(triggerName, args...).Output() +} + +// PlugnTriggerSetup sets up a plugn trigger call +func PlugnTriggerSetup(triggerName string, args ...string) *sh.Session { + shellArgs := make([]interface{}, len(args)+2) + shellArgs[0] = "trigger" + shellArgs[1] = triggerName + for i, arg := range args { + shellArgs[i+2] = arg + } + return sh.Command("plugn", shellArgs...) +} From 39984980fb167de691b6c8bd973449062cba824a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 19:49:39 -0400 Subject: [PATCH 11/67] fix: correct io tests --- plugins/common/io_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/common/io_test.go b/plugins/common/io_test.go index 856206a87..e03ad046a 100644 --- a/plugins/common/io_test.go +++ b/plugins/common/io_test.go @@ -1,5 +1,10 @@ +package common -import "testing" +import ( + "testing" + + . "github.com/onsi/gomega" +) func TestCommonFileToSlice(t *testing.T) { RegisterTestingT(t) From 3714a231a99606f8ec680612b9bedbe1ff5e9993 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 19:49:58 -0400 Subject: [PATCH 12/67] chore: drop unused code --- plugins/ps/functions | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/ps/functions b/plugins/ps/functions index bec1dbfbb..fec5beb55 100755 --- a/plugins/ps/functions +++ b/plugins/ps/functions @@ -88,9 +88,6 @@ generate_scale_file() { if [[ ! -f $DOKKU_SCALE_FILE ]]; then dokku_log_info1_quiet "DOKKU_SCALE file not found in app image. Generating one based on Procfile..." - local GENERATE_SCALE_FILE_TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") - trap "rm -rf '$GENERATE_SCALE_FILE_TMP_WORK_DIR' >/dev/null" RETURN - if [[ -f $DOKKU_PROCFILE ]]; then while read -r line || [[ -n "$line" ]]; do if [[ -z "$line" ]] || [[ "$line" == "#"* ]]; then From 947b2d4db6c8ed6ac7c229389055542e7a9a1d73 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 10 Oct 2020 19:51:00 -0400 Subject: [PATCH 13/67] fix: add missing comment --- plugins/common/io.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/common/io.go b/plugins/common/io.go index f57b20e42..79e98452b 100644 --- a/plugins/common/io.go +++ b/plugins/common/io.go @@ -170,6 +170,7 @@ func SetPermissions(path string, fileMode os.FileMode) error { return os.Chown(path, uid, gid) } +// WriteSliceToFile writes a slice of strings to a file func WriteSliceToFile(filename string, lines []string) error { file, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC, 0600) if err != nil { From e0a70411dd82ccce5b656dd51f39077b41fd267a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 15:50:08 -0500 Subject: [PATCH 14/67] fix: create the file if it does not exist --- plugins/common/io.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/common/io.go b/plugins/common/io.go index 79e98452b..2918284f8 100644 --- a/plugins/common/io.go +++ b/plugins/common/io.go @@ -172,7 +172,7 @@ func SetPermissions(path string, fileMode os.FileMode) error { // WriteSliceToFile writes a slice of strings to a file func WriteSliceToFile(filename string, lines []string) error { - file, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC, 0600) + file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return err } From 34343a846d3e97e49a00b0a473c450b6ba736150 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 15:51:24 -0500 Subject: [PATCH 15/67] chore: drop alias --- plugins/common/common.go | 2 +- plugins/proxy/proxy.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index fdd53aee0..50085d29f 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -12,7 +12,7 @@ import ( "strings" "unicode" - columnize "github.com/ryanuber/columnize" + "github.com/ryanuber/columnize" ) type errfunc func() error diff --git a/plugins/proxy/proxy.go b/plugins/proxy/proxy.go index 57fc08e9e..0a1860d2b 100644 --- a/plugins/proxy/proxy.go +++ b/plugins/proxy/proxy.go @@ -10,7 +10,7 @@ import ( "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" - columnize "github.com/ryanuber/columnize" + "github.com/ryanuber/columnize" ) // PortMap is a struct that contains a scheme:host-port:container-port mapping From c72bbf41ea1d691df8c0d5c0f771fc9cffb9a173 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 15:51:41 -0500 Subject: [PATCH 16/67] chore: use helper method --- plugins/repo/repo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/repo/repo.go b/plugins/repo/repo.go index ce7813cd8..938602077 100644 --- a/plugins/repo/repo.go +++ b/plugins/repo/repo.go @@ -16,8 +16,8 @@ func PurgeCache(appName string) error { return err } - cacheDir := strings.Join([]string{common.MustGetEnv("DOKKU_ROOT"), appName, "cache"}, "/") - cacheHostDir := strings.Join([]string{common.MustGetEnv("DOKKU_HOST_ROOT"), appName, "cache"}, "/") + cacheDir := strings.Join([]string{common.AppRoot(appName), "cache"}, "/") + cacheHostDir := strings.Join([]string{common.AppHostRoot(appName), "cache"}, "/") dokkuGlobalRunArgs := common.MustGetEnv("DOKKU_GLOBAL_RUN_ARGS") image := config.GetWithDefault(appName, "DOKKU_IMAGE", os.Getenv("DOKKU_IMAGE")) if info, _ := os.Stat(cacheDir); info != nil && info.IsDir() { From e7ee7a17815ca9d23e8ef8d34c73dce8c7d9ab41 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 15:53:53 -0500 Subject: [PATCH 17/67] refactor: implement most of ps plugin in golang --- docs/appendices/0.22.0-migration-guide.md | 12 + docs/deployment/process-management.md | 60 +---- plugins/common/common.go | 11 + plugins/docker-options/dockeroptions.go | 58 ++--- plugins/docker-options/functions.go | 36 +++ plugins/ps/.gitignore | 11 + plugins/ps/Makefile | 6 + plugins/ps/app-restart | 15 -- plugins/ps/commands | 16 -- plugins/ps/core-post-deploy | 17 -- plugins/ps/functions.go | 287 ++++++++++++++++++++++ plugins/ps/go.mod | 15 ++ plugins/ps/go.sum | 10 + plugins/ps/help-functions | 45 ---- plugins/ps/install | 26 -- plugins/ps/post-app-clone | 18 -- plugins/ps/post-app-rename | 18 -- plugins/ps/post-create | 20 -- plugins/ps/post-extract | 24 -- plugins/ps/post-stop | 16 -- plugins/ps/pre-deploy | 16 -- plugins/ps/procfile-extract | 25 -- plugins/ps/procfile-get-command | 19 -- plugins/ps/procfile-remove | 22 -- plugins/ps/ps.go | 148 +++++++++++ plugins/ps/report | 6 - plugins/ps/src/commands/commands.go | 62 +++++ plugins/ps/src/subcommands/subcommands.go | 92 +++++++ plugins/ps/src/triggers/triggers.go | 80 ++++++ plugins/ps/subcommands.go | 215 ++++++++++++++++ plugins/ps/subcommands/default | 6 - plugins/ps/subcommands/inspect | 6 - plugins/ps/subcommands/rebuild | 22 -- plugins/ps/subcommands/rebuildall | 15 -- plugins/ps/subcommands/report | 6 - plugins/ps/subcommands/restart | 16 -- plugins/ps/subcommands/restart-policy | 22 -- plugins/ps/subcommands/restartall | 15 -- plugins/ps/subcommands/restore | 40 --- plugins/ps/subcommands/retire | 20 -- plugins/ps/subcommands/scale | 17 -- plugins/ps/subcommands/set-restart-policy | 34 --- plugins/ps/subcommands/start | 17 -- plugins/ps/subcommands/startall | 15 -- plugins/ps/subcommands/stop | 17 -- plugins/ps/subcommands/stopall | 15 -- plugins/ps/triggers.go | 193 +++++++++++++++ tests/unit/ps-general.bats | 21 +- 48 files changed, 1227 insertions(+), 676 deletions(-) create mode 100644 plugins/docker-options/functions.go create mode 100644 plugins/ps/.gitignore create mode 100644 plugins/ps/Makefile delete mode 100755 plugins/ps/app-restart delete mode 100755 plugins/ps/commands delete mode 100755 plugins/ps/core-post-deploy create mode 100644 plugins/ps/functions.go create mode 100644 plugins/ps/go.mod create mode 100644 plugins/ps/go.sum delete mode 100755 plugins/ps/help-functions delete mode 100755 plugins/ps/install delete mode 100755 plugins/ps/post-app-clone delete mode 100755 plugins/ps/post-app-rename delete mode 100755 plugins/ps/post-create delete mode 100755 plugins/ps/post-extract delete mode 100755 plugins/ps/post-stop delete mode 100755 plugins/ps/pre-deploy delete mode 100755 plugins/ps/procfile-extract delete mode 100755 plugins/ps/procfile-get-command delete mode 100755 plugins/ps/procfile-remove create mode 100644 plugins/ps/ps.go delete mode 100755 plugins/ps/report create mode 100644 plugins/ps/src/commands/commands.go create mode 100644 plugins/ps/src/subcommands/subcommands.go create mode 100644 plugins/ps/src/triggers/triggers.go create mode 100644 plugins/ps/subcommands.go delete mode 100755 plugins/ps/subcommands/default delete mode 100755 plugins/ps/subcommands/inspect delete mode 100755 plugins/ps/subcommands/rebuild delete mode 100755 plugins/ps/subcommands/rebuildall delete mode 100755 plugins/ps/subcommands/report delete mode 100755 plugins/ps/subcommands/restart delete mode 100755 plugins/ps/subcommands/restart-policy delete mode 100755 plugins/ps/subcommands/restartall delete mode 100755 plugins/ps/subcommands/restore delete mode 100755 plugins/ps/subcommands/retire delete mode 100755 plugins/ps/subcommands/scale delete mode 100755 plugins/ps/subcommands/set-restart-policy delete mode 100755 plugins/ps/subcommands/start delete mode 100755 plugins/ps/subcommands/startall delete mode 100755 plugins/ps/subcommands/stop delete mode 100755 plugins/ps/subcommands/stopall create mode 100644 plugins/ps/triggers.go diff --git a/docs/appendices/0.22.0-migration-guide.md b/docs/appendices/0.22.0-migration-guide.md index 55f04a508..ca5b96cbe 100644 --- a/docs/appendices/0.22.0-migration-guide.md +++ b/docs/appendices/0.22.0-migration-guide.md @@ -5,3 +5,15 @@ - Underscores are no longer valid characters in app names. Please rename applications before upgrading. - Process type names specified in Procfile may no longer use characters not valid in DNS Label Names ([RFC 1123](https://tools.ietf.org/html/rfc1123)). - The minimum Docker version is now 17.05.0. + +## Removals + +The `ps` command has had a few removals as a result of a rewrite to Golang: + +- The `ps:set-restart-policy` command has been removed in favor the `ps:set` command. +- The `ps:restart-policy` command has been removed in favor of the `ps:report` command. +- The `ps:rebuildall` command has been removed in favor of calling the `ps:rebuild` with the `--all` flag instead of an app name. +- The `ps:restartall` command has been removed in favor of calling the `ps:restart` with the `--all` flag instead of an app name. +- The `ps:stopall` command has been removed in favor of calling the `ps:stop` with the `--all` flag instead of an app name. +- The `ps:startall` command has been removed in favor of calling the `ps:start` with the `--all` flag instead of an app name. +- The `DOKKU_PROCFILE` file is no longer located in the git directory for a given app on the server. It's location is currently not exposed. diff --git a/docs/deployment/process-management.md b/docs/deployment/process-management.md index f44b3351e..3478b608f 100644 --- a/docs/deployment/process-management.md +++ b/docs/deployment/process-management.md @@ -8,10 +8,9 @@ ps:rebuild # Rebuild an app from source ps:rebuildall # Rebuild all apps from source ps:report [] [] # Displays a process report for one or more apps ps:restart # Restart app container(s) -ps:restart-policy # Shows the restart-policy for an app ps:restartall # Restart all deployed app containers ps:scale = [=] # Get/Set how many instances of a given process to run -ps:set-restart-policy # Sets app restart-policy +ps:set # Set or clear a ps property for an app ps:start # Start app container(s) ps:startall # Start all deployed app containers ps:stop # Stop app container(s) @@ -133,68 +132,29 @@ dokku ps:startall > New as of 0.7.0 -By default, Dokku will automatically restart containers that exit with a non-zero status up to 10 times via the [on-failure Docker restart policy](https://docs.docker.com/engine/reference/run/#restart-policies-restart). - -### Showing the current restart policy - -The `ps:restart-policy` command will show the currently configured restart policy for an application. The default policy is `on-failure:10` - -```shell -dokku ps:restart-policy node-js-app -``` - -``` -=====> node-js-app restart-policy: -on-failure:10 -``` +By default, Dokku will automatically restart containers that exit with a non-zero status up to 10 times via the [on-failure Docker restart policy](https://docs.docker.com/engine/reference/run/#restart-policies---restart). ### Setting the restart policy -You can configure this via the `ps:set-restart-policy` command: +> A change in the restart policy must be followed by a `ps:rebuild` call. + +You can configure this via the `ps:set` command: ```shell # always restart an exited container -dokku ps:set-restart-policy node-js-app always -``` +dokku ps:set node-js-app restart-policy always -``` ------> Setting restart policy: always -``` - -```shell # never restart an exited container -dokku ps:set-restart-policy node-js-app no -``` +dokku ps:set node-js-app restart-policy no -``` ------> Setting restart policy: no -``` - -```shell # only restart it on Docker restart if it was not manually stopped -dokku ps:set-restart-policy node-js-app unless-stopped -``` +dokku ps:set node-js-app restart-policy unless-stopped -``` ------> Setting restart policy: unless-stopped -``` - -```shell # restart only on non-zero exit status -dokku ps:set-restart-policy node-js-app on-failure -``` +dokku ps:set node-js-app restart-policy on-failure -``` ------> Setting restart policy: on-failure -``` - -```shell # restart only on non-zero exit status up to 20 times -dokku ps:set-restart-policy node-js-app on-failure:20 -``` - -``` ------> Setting restart policy: on-failure:20 +dokku ps:set node-js-app restart-policy on-failure:20 ``` Restart policies have no bearing on server reboot, and Dokku will always attempt to restart your applications at that point unless they were manually stopped. diff --git a/plugins/common/common.go b/plugins/common/common.go index 50085d29f..be763e55a 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -9,6 +9,7 @@ import ( "reflect" "regexp" "sort" + "strconv" "strings" "unicode" @@ -379,6 +380,16 @@ func ToBool(s string) bool { return s == "true" } +// ToInt returns an int value for a given string +func ToInt(s string, defaultValue int) int { + i, err := strconv.Atoi(s) + if err != nil { + return defaultValue + } + + return i +} + // UcFirst uppercases the first character in a string func UcFirst(str string) string { for i, v := range str { diff --git a/plugins/docker-options/dockeroptions.go b/plugins/docker-options/dockeroptions.go index a1d44e6b3..1a30faa25 100644 --- a/plugins/docker-options/dockeroptions.go +++ b/plugins/docker-options/dockeroptions.go @@ -4,13 +4,39 @@ import ( "bufio" "fmt" "os" - "path/filepath" "sort" "strings" - - "github.com/dokku/dokku/plugins/common" ) +func SetDockerOptionForPhases(appName string, phases []string, name string, value string) error { + for _, phase := range phases { + if err := touchPhaseFile(appName, phase); err != nil { + return err + } + + options, err := GetDockerOptionsForPhase(appName, phase) + if err != nil { + return err + } + + newOptions := []string{} + for _, option := range options { + if strings.HasPrefix(option, fmt.Sprintf("--%s=", name)) { + continue + } + + newOptions = append(newOptions, option) + } + + newOptions = append(newOptions, fmt.Sprintf("--%s=%s", name, value)) + sort.Strings(newOptions) + if err = writeDockerOptionsForPhase(appName, phase, newOptions); err != nil { + return err + } + } + return nil +} + // AddDockerOptionToPhases adds docker option to specified phases func AddDockerOptionToPhases(appName string, phases []string, option string) error { for _, phase := range phases { @@ -58,29 +84,3 @@ func GetDockerOptionsForPhase(appName string, phase string) ([]string, error) { return options, nil } - -func getPhaseFilePath(appName string, phase string) string { - return filepath.Join(common.MustGetEnv("DOKKU_ROOT"), appName, "DOCKER_OPTIONS_"+strings.ToUpper(phase)) -} - -func touchPhaseFile(appName string, phase string) error { - phaseFilePath := getPhaseFilePath(appName, phase) - - _, err := os.Stat(phaseFilePath) - if !os.IsNotExist(err) { - return nil - } - - file, err := os.Create(phaseFilePath) - if err != nil { - return fmt.Errorf("Unable to create docker options phase file %s.%s: %s", appName, phase, err.Error()) - } - defer file.Close() - - return nil -} - -func writeDockerOptionsForPhase(appName string, phase string, options []string) error { - phaseFilePath := getPhaseFilePath(appName, phase) - return common.WriteSliceToFile(phaseFilePath, options) -} diff --git a/plugins/docker-options/functions.go b/plugins/docker-options/functions.go new file mode 100644 index 000000000..d7f1919e6 --- /dev/null +++ b/plugins/docker-options/functions.go @@ -0,0 +1,36 @@ +package dockeroptions + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +func getPhaseFilePath(appName string, phase string) string { + return filepath.Join(common.AppRoot(appName), "DOCKER_OPTIONS_"+strings.ToUpper(phase)) +} + +func touchPhaseFile(appName string, phase string) error { + phaseFilePath := getPhaseFilePath(appName, phase) + + _, err := os.Stat(phaseFilePath) + if !os.IsNotExist(err) { + return nil + } + + file, err := os.Create(phaseFilePath) + if err != nil { + return fmt.Errorf("Unable to create docker options phase file %s.%s: %s", appName, phase, err.Error()) + } + defer file.Close() + + return nil +} + +func writeDockerOptionsForPhase(appName string, phase string, options []string) error { + phaseFilePath := getPhaseFilePath(appName, phase) + return common.WriteSliceToFile(phaseFilePath, options) +} diff --git a/plugins/ps/.gitignore b/plugins/ps/.gitignore new file mode 100644 index 000000000..3922c71ac --- /dev/null +++ b/plugins/ps/.gitignore @@ -0,0 +1,11 @@ +/commands +/subcommands/* +/triggers/* +/triggers +/proxy-* +/install +/post-* +/report +/pre-* +/procfile-* +/app-restart diff --git a/plugins/ps/Makefile b/plugins/ps/Makefile new file mode 100644 index 000000000..004322380 --- /dev/null +++ b/plugins/ps/Makefile @@ -0,0 +1,6 @@ +SUBCOMMANDS = subcommands/inspect subcommands/rebuild subcommands/rebuildall subcommands/report subcommands/restart subcommands/restart-policy subcommands/restartall subcommands/restore subcommands/retire subcommands/scale subcommands/set subcommands/start subcommands/startall subcommands/stop subcommands/stopall +TRIGGERS = triggers/app-restart triggers/install triggers/post-app-clone triggers/post-app-rename triggers/post-create triggers/post-delete triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/report +BUILD = commands subcommands triggers +PLUGIN_NAME = ps + +include ../../common.mk diff --git a/plugins/ps/app-restart b/plugins/ps/app-restart deleted file mode 100755 index 064bfd845..000000000 --- a/plugins/ps/app-restart +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-app-restart() { - declare desc="ps app-restart plugin trigger" - declare trigger="app-restart" - declare APP="$1" - - ps_restart "$APP" -} - -trigger-ps-app-restart "$@" diff --git a/plugins/ps/commands b/plugins/ps/commands deleted file mode 100755 index 2f2be37cc..000000000 --- a/plugins/ps/commands +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -[[ " help ps:help " == *" $1 "* ]] || exit "$DOKKU_NOT_IMPLEMENTED_EXIT" -source "$PLUGIN_AVAILABLE_PATH/ps/help-functions" -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x - -case "$1" in - help | ps:help) - cmd-ps-help "$@" - ;; - - *) - exit "$DOKKU_NOT_IMPLEMENTED_EXIT" - ;; - -esac diff --git a/plugins/ps/core-post-deploy b/plugins/ps/core-post-deploy deleted file mode 100755 index b789ddc20..000000000 --- a/plugins/ps/core-post-deploy +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-core-post-deploy() { - declare desc="ps core-post-deploy plugin trigger" - declare trigger="core-post-deploy" - declare APP="$1" - - remove_procfile "$APP" - DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$APP" DOKKU_APP_RESTORE=1 -} - -trigger-ps-core-post-deploy "$@" diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go new file mode 100644 index 000000000..45fcfa151 --- /dev/null +++ b/plugins/ps/functions.go @@ -0,0 +1,287 @@ +package ps + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/dokku/dokku/plugins/common" + dockeroptions "github.com/dokku/dokku/plugins/docker-options" +) + +func canScaleApp(appName string) bool { + return !common.FileExists(getScalefilePath(appName)) +} + +func extractProcfile(appName, image string, procfilePath string) error { + if err := removeProcfile(appName); err != nil { + return err + } + + destination := getProcfilePath(appName) + common.CopyFromImage(appName, image, "Procfile", destination) + if !common.FileExists(destination) { + common.LogInfo1Quiet("No Procfile found in app image") + return nil + } + + common.LogInfo1Quiet("App Procfile file found") + checkCmd := common.NewShellCmd(strings.Join([]string{ + "procfile-util", + "check", + "--procfile", + destination, + }, " ")) + var stderr bytes.Buffer + checkCmd.ShowOutput = false + checkCmd.Command.Stderr = &stderr + _, err := checkCmd.Output() + + if err != nil { + return fmt.Errorf(strings.TrimSpace(stderr.String())) + } + + return nil +} + + +func extractOrGenerateScalefile(appName string, image string) error { + destination := getScalefilePath(appName) + extracted := getScalefileExtractedPath(appName) + if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", destination); err != nil { + os.Remove(extracted) + os.Remove(destination) + } else if err := common.CopyFile(destination, extracted); err != nil { + return err + } + + if common.FileExists(destination) { + return nil + } + + common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") + if err := generateScalefile(appName, destination); err != nil { + common.LogDebug(fmt.Sprintf("Error: %s", err.Error())) + return err + } + return nil +} + +func generateScalefile(appName string, destination string) error { + procfilePath := getProcfilePath(appName) + content := []string{"web=1"} + if !common.FileExists(procfilePath) { + return common.WriteSliceToFile(destination, content) + } + + lines, err := common.FileToSlice(procfilePath) + if err != nil { + return common.WriteSliceToFile(destination, content) + } + + content = []string{} + for _, line := range lines { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.Split(line, ":") + count := 0 + if parts[0] == "web" { + count = 1 + } + + content = append(content, fmt.Sprintf("%s=%d", parts[0], count)) + } + + return common.WriteSliceToFile(destination, content) +} + +func getProcfileCommand(procfilePath string, processType string, port int) (string, error) { + shellCmd := common.NewShellCmd(strings.Join([]string{ + "procfile-util", + "show", + "--procfile", + procfilePath, + "--process-type", + processType, + "--default-port", + strconv.Itoa(port), + }, " ")) + var stderr bytes.Buffer + shellCmd.ShowOutput = false + shellCmd.Command.Stderr = &stderr + b, err := shellCmd.Output() + + if err != nil { + return "", fmt.Errorf(strings.TrimSpace(stderr.String())) + } + + return strings.TrimSpace(string(b[:])), nil +} + +func getProcfilePath(appName string) string { + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", appName) + return filepath.Join(directory, "Procfile") +} + +func getRestartPolicy(appName string) (string, error) { + options, err := dockeroptions.GetDockerOptionsForPhase(appName, "deploy") + if err != nil { + return "", err + } + + for _, option := range options { + if strings.HasPrefix(option, "--restart=") { + return strings.TrimPrefix(option, "--restart="), nil + } + } + + return "", nil +} + +func getRunningState(appName string) string { + scheduler := common.GetAppScheduler(appName) + b, _ := common.PlugnTriggerOutput("scheduler-app-status", []string{scheduler, appName}...) + return strings.Split(strings.TrimSpace(string(b[:])), " ")[1] +} + +func getScalefilePath(appName string) string { + return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE") +} + +func getScalefileExtractedPath(appName string) string { + return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") +} + +func isValidRestartPolicy(policy string) bool { + if policy == "" { + return false + } + + validRestartPolicies := map[string]bool{ + "no": true, + "always": true, + "unless-stopped": true, + "on-failure": true, + } + + if _, ok := validRestartPolicies[policy]; ok { + return true + } + + return strings.HasPrefix(policy, "on-failure:") +} + +func processesInProcfile(procfilePath string) (map[string]bool, error) { + processes := map[string]bool{} + + shellCmd := common.NewShellCmd(strings.Join([]string{ + "procfile-util", + "list", + "--procfile", + procfilePath, + }, " ")) + var stderr bytes.Buffer + shellCmd.ShowOutput = false + shellCmd.Command.Stderr = &stderr + b, err := shellCmd.Output() + + if err != nil { + return processes, fmt.Errorf(strings.TrimSpace(stderr.String())) + } + + for _, s := range strings.Split(strings.TrimSpace(string(b[:])), "\n") { + processes[s] = true + } + + return processes, nil +} + +func removeProcfile(appName string) error { + procfile := getProcfilePath(appName) + if !common.FileExists(procfile) { + return nil + } + + return os.Remove(procfile) +} + +func updateScalefile(appName string, processTuples []string) error { + procfilePath := getProcfilePath(appName) + scalefilePath := getScalefilePath(appName) + lines, err := common.FileToSlice(scalefilePath) + if err != nil { + return err + } + + processTypes, err := processesInProcfile(procfilePath) + if err != nil { + return err + } + + newLines := []string{} + for processType := range processTypes { + found := false + for _, line := range lines { + if strings.HasPrefix(line, fmt.Sprintf("%s=", processType)) { + newLines = append(newLines, line) + found = true + break + } + } + + if !found { + newLines = append(newLines, fmt.Sprintf("%s=0", processType)) + } + } + + scale := []string{} + for _, processTuple := range processTuples { + s := strings.Split(processTuple, "=") + if len(s) == 1 { + return fmt.Errorf("Missing count for process type %s", processTuple) + } + + processType := s[0] + count, err := strconv.Atoi(s[1]) + if err != nil { + return fmt.Errorf("Invalid count for process type %s", s[0]) + } + + if _, ok := processTypes[processType]; !ok { + if count != 0 && len(processTypes) == 0 { + return fmt.Errorf("%s is not a valid process name to scale up", processType) + } + } + + scale = append(scale, fmt.Sprintf("%s=%d", processType, count)) + } + + for _, line := range newLines { + s := strings.Split(line, "=") + processType := s[0] + + found := false + for _, s := range scale { + if strings.HasPrefix(s, fmt.Sprintf("%s=", processType)) { + found = true + break + } + } + + if !found { + scale = append(scale, line) + } + + } + + if err := common.WriteSliceToFile(scalefilePath, scale); err != nil { + return err + } + + return err +} diff --git a/plugins/ps/go.mod b/plugins/ps/go.mod new file mode 100644 index 000000000..8cb14fd8a --- /dev/null +++ b/plugins/ps/go.mod @@ -0,0 +1,15 @@ +module github.com/dokku/dokku/plugins/ps + +go 1.14 + +require ( + github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 + github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 + github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000 + github.com/dokku/dokku/plugins/docker-options v0.0.0-00010101000000-000000000000 + github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 +) + +replace github.com/dokku/dokku/plugins/common => ../common +replace github.com/dokku/dokku/plugins/config => ../config +replace github.com/dokku/dokku/plugins/docker-options => ../docker-options diff --git a/plugins/ps/go.sum b/plugins/ps/go.sum new file mode 100644 index 000000000..8b973f334 --- /dev/null +++ b/plugins/ps/go.sum @@ -0,0 +1,10 @@ +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= +github.com/dokku/dokku v0.21.4 h1:WG0hyPzNqKqoQXuI7H/N+cWX9B7mAxTW6Yop40BjCQk= +github.com/joho/godotenv v1.2.0 h1:vGTvz69FzUFp+X4/bAkb0j5BoLC+9bpqTWY8mjhA9pc= +github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= diff --git a/plugins/ps/help-functions b/plugins/ps/help-functions deleted file mode 100755 index df2913503..000000000 --- a/plugins/ps/help-functions +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x - -cmd-ps-help() { - declare desc="help command" - declare CMD="$1" - local plugin_name="ps" - local plugin_description="Manage container processes" - - if [[ "$CMD" == "$plugin_name:help" ]]; then - echo -e "Usage: dokku $plugin_name[:COMMAND]" - echo '' - echo "$plugin_description" - echo '' - echo 'Additional commands:' - fn-help-content | sort | column -c2 -t -s, - elif [[ $(ps -o command= $PPID) == *"--all"* ]]; then - fn-help-content - else - cat <, Displays a sanitized version of docker inspect for an app - ps:scale = [=...], Get/Set how many instances of a given process to run - ps:start , Start app container(s) - ps:startall, Starts all apps via command line - ps:stop , Stop app container(s) - ps:stopall, Stop all app container(s) - ps:rebuild , Rebuild an app from source - ps:rebuildall, Rebuild all apps from source - ps:report [] [], Displays a process report for one or more apps - ps:restart , Restart app container(s) - ps:restartall, Restart all deployed app containers - ps:restore, Start previously running apps e.g. after reboot - ps:restart-policy , Shows the restart-policy for an app - ps:set-restart-policy , Sets app restart-policy -help_content -} diff --git a/plugins/ps/install b/plugins/ps/install deleted file mode 100755 index 4d57b90be..000000000 --- a/plugins/ps/install +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/docker-options/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-install() { - declare desc="set the default restart policy for all applications if there is not one already set" - declare trigger="install" - local APPS="$(dokku_apps)" - local APP - - mkdir -p "${DOKKU_LIB_ROOT}/data/ps" - chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/ps" - - for APP in $APPS; do - local RESTART_POLICIES=$(get_restart_policies "$(get_phase_file_path "deploy")") - if [[ -z "$RESTART_POLICIES" ]]; then - local passed_phases=(deploy) - add_passed_docker_option passed_phases[@] "--restart=on-failure:10" - fi - done -} - -trigger-ps-install "$@" diff --git a/plugins/ps/post-app-clone b/plugins/ps/post-app-clone deleted file mode 100755 index 92fdefd7b..000000000 --- a/plugins/ps/post-app-clone +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-post-app-clone() { - declare desc="ps post-app-clone plugin trigger" - declare trigger="post-app-clone" - declare OLD_APP="$1" NEW_APP="$2" - - if [[ "$SKIP_REBUILD" != "true" ]]; then - ps_rebuild "$NEW_APP" - fi -} - -trigger-ps-post-app-clone "$@" diff --git a/plugins/ps/post-app-rename b/plugins/ps/post-app-rename deleted file mode 100755 index 7175b517b..000000000 --- a/plugins/ps/post-app-rename +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-post-app-rename() { - declare desc="ps post-app-rename plugin trigger" - declare trigger="post-app-rename" - declare OLD_APP="$1" NEW_APP="$2" - - if [[ "$SKIP_REBUILD" != "true" ]]; then - ps_rebuild "$NEW_APP" - fi -} - -trigger-ps-post-app-rename "$@" diff --git a/plugins/ps/post-create b/plugins/ps/post-create deleted file mode 100755 index 14efa77a8..000000000 --- a/plugins/ps/post-create +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/docker-options/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-post-create() { - declare desc="ps post-create plugin trigger" - declare trigger="post-create" - local APP="$1" - - local passed_phases=(deploy) - add_passed_docker_option passed_phases[@] "--restart=on-failure:10" - - mkdir -p "${DOKKU_LIB_ROOT}/data/ps/${APP}" - chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/ps/${APP}" -} - -trigger-ps-post-create "$@" diff --git a/plugins/ps/post-extract b/plugins/ps/post-extract deleted file mode 100755 index ac329ccfd..000000000 --- a/plugins/ps/post-extract +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" - -trigger-ps-post-extract() { - declare desc="ps post-extract plugin trigger" - declare trigger="post-extract" - declare APP="$1" TMP_WORK_DIR="$2" REV="$3" - - if [[ ! -f "$TMP_WORK_DIR/Procfile" ]]; then - return - fi - - pushd "$TMP_WORK_DIR" >/dev/null - for process_type in $(procfile-util list); do - if echo "$process_type" | grep -vqE '^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$'; then - dokku_log_fail "Invalid process type name ($process_type)" - fi - done - popd >/dev/null -} - -trigger-ps-post-extract "$@" diff --git a/plugins/ps/post-stop b/plugins/ps/post-stop deleted file mode 100755 index 4d091c180..000000000 --- a/plugins/ps/post-stop +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-post-stop() { - declare desc="ps post-stop plugin trigger" - declare trigger="post-stop" - declare APP="$1" - - DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$APP" DOKKU_APP_RESTORE=0 -} - -trigger-ps-post-stop "$@" diff --git a/plugins/ps/pre-deploy b/plugins/ps/pre-deploy deleted file mode 100755 index 4e4e85db8..000000000 --- a/plugins/ps/pre-deploy +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-pre-deploy() { - declare desc="ps pre-deploy plugin trigger" - declare trigger="pre-deploy" - declare APP="$1" IMAGE_TAG="$2" - - extract_procfile "$APP" "$IMAGE_TAG" - generate_scale_file "$APP" "$IMAGE_TAG" -} - -trigger-ps-pre-deploy "$@" diff --git a/plugins/ps/procfile-extract b/plugins/ps/procfile-extract deleted file mode 100755 index 74a2ec218..000000000 --- a/plugins/ps/procfile-extract +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-procfile-extract() { - declare desc="ps procfile-extract plugin trigger" - declare trigger="procfile-extract" - declare desc="extract procfile from a specified docker image" - declare APP="$1" IMAGE="$2" FORCE_EXTRACT="$3" PROCFILE_PATH="$4" - - if [[ -z "$PROCFILE_PATH" ]]; then - PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" - mkdir -p "${DOKKU_LIB_ROOT}/data/ps/${APP}" - chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/ps/${APP}" - fi - - if [[ "$FORCE_EXTRACT" == "true" ]]; then - plugn trigger procfile-remove "$APP" "$PROCFILE_PATH" - fi - - fn-procfile-extract "$IMAGE" "$PROCFILE_PATH" -} - -trigger-ps-procfile-extract "$@" diff --git a/plugins/ps/procfile-get-command b/plugins/ps/procfile-get-command deleted file mode 100755 index 00c96d6a6..000000000 --- a/plugins/ps/procfile-get-command +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -trigger-ps-procfile-get-command() { - declare desc="ps procfile-get-command plugin trigger" - declare trigger="procfile-get-command" - declare desc="extract command from an app procfile" - declare APP="$1" PROCESS_TYPE="$2" PORT="$3" PROCFILE_PATH="$4" - - if [[ -z "$PROCFILE_PATH" ]]; then - PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" - fi - - fn-procfile-get-command "$PROCFILE_PATH" "$PROCESS_TYPE" "$PORT" -} - -trigger-ps-procfile-get-command "$@" diff --git a/plugins/ps/procfile-remove b/plugins/ps/procfile-remove deleted file mode 100755 index 28b1ced70..000000000 --- a/plugins/ps/procfile-remove +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x - -trigger-ps-procfile-remove() { - declare desc="ps procfile-remove plugin trigger" - declare trigger="procfile-remove" - declare desc="remove procfile for a given app" - declare APP="$1" PROCFILE_PATH="$2" - - if [[ -z "$PROCFILE_PATH" ]]; then - PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" - fi - - if [[ ! -f "$PROCFILE_PATH" ]]; then - return - fi - - rm -f "$PROCFILE_PATH" 2>/dev/null -} - -trigger-ps-procfile-remove "$@" diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go new file mode 100644 index 000000000..a4e7b1db9 --- /dev/null +++ b/plugins/ps/ps.go @@ -0,0 +1,148 @@ +package ps + +import ( + "fmt" + + "github.com/dokku/dokku/plugins/common" +) + +const RUN_IN_SERIAL = -1 + +type ParallelCommand func(string) error + +var ( + // DefaultProperties is a map of all valid ps properties with corresponding default property values + DefaultProperties = map[string]string{ + "restart-policy": "on-failure:10", + } +) + +func RunCommandAgainstAllApps(command ParallelCommand, commandName string, runInSerial bool, parallelCount int) error { + if runInSerial && parallelCount != RUN_IN_SERIAL { + common.LogWarn("Ignoring --parallel value and running in serial mode") + } + + if runInSerial { + return RunCommandAgainstAllAppsSerially(command, commandName) + } + + return RunCommandAgainstAllAppsInParallel(command, commandName, parallelCount) +} + +// TODO: implement me +func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error { + return nil +} + +func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error { + apps, err := common.DokkuApps() + if err != nil { + common.LogWarn(err.Error()) + return nil + } + + errorCount := 0 + for _, appName := range apps { + if err = command(appName); err != nil { + errorCount += 1 + } + } + + if errorCount > 0 { + return fmt.Errorf("%s command returned %d errors", commandName, errorCount) + } + + return nil +} + +// Rebuild rebuilds app from base image +func Rebuild(appName string) error { + return common.PlugnTrigger("receive-app", []string{appName}...) +} + +// ReportSingleApp is an internal function that displays the ps report for one or more apps +// TODO: implement me +func ReportSingleApp(appName, infoFlag string) error { + if err := common.VerifyAppName(appName); err != nil { + return err + } + + + policy, _ := getRestartPolicy(appName) + if policy == "" { + policy = DefaultProperties["restart-policy"] + } + + infoFlags := map[string]string{ + "--ps-restart-policy": policy, + } + + trimPrefix := false + uppercaseFirstCharacter := true + return common.ReportSingleApp("ps", appName, infoFlag, infoFlags, trimPrefix, uppercaseFirstCharacter) +} + +// Restart restarts the app +func Restart(appName string) error { + if !common.IsDeployed(appName) { + common.LogWarn(fmt.Sprintf("App %s has not been deployed", appName)) + return nil + } + + return common.PlugnTrigger("release-and-deploy", []string{appName}...) +} + +// Start starts the app +func Start(appName string) error { + imageTag, _ := common.GetRunningImageTag(appName) + + if !common.IsDeployed(appName) { + common.LogWarn(fmt.Sprintf("App %s has not been deployed", appName)) + return nil + } + + if err := common.PlugnTrigger("pre-start", []string{appName}...); err != nil { + return fmt.Errorf("Failure in pre-start hook: %s", err) + } + + runningState := getRunningState(appName) + + if runningState == "mixed" { + common.LogWarn("App is running in mixed mode, releasing") + } + + if runningState != "true" { + if err := common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...); err != nil { + return err + } + } else { + common.LogWarn(fmt.Sprintf("App %s already running", appName)) + } + + if err := common.PlugnTrigger("proxy-build-config", []string{appName}...); err != nil { + return err + } + + return nil +} + +// Stop stops the app +func Stop(appName string) error { + if !common.IsDeployed(appName) { + common.LogWarn(fmt.Sprintf("App %s has not been deployed", appName)) + return nil + } + + common.LogQuiet(fmt.Sprintf("Stopping %s", appName)) + scheduler := common.GetAppScheduler(appName) + + if err := common.PlugnTrigger("scheduler-stop", []string{scheduler, appName}...); err != nil { + return err + } + + if err := common.PlugnTrigger("post-stop", []string{appName}...); err != nil { + return err + } + + return nil +} diff --git a/plugins/ps/report b/plugins/ps/report deleted file mode 100755 index 273d4431c..000000000 --- a/plugins/ps/report +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x - -cmd-ps-report-single "$@" diff --git a/plugins/ps/src/commands/commands.go b/plugins/ps/src/commands/commands.go new file mode 100644 index 000000000..259073d65 --- /dev/null +++ b/plugins/ps/src/commands/commands.go @@ -0,0 +1,62 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +const ( + helpHeader = `Usage: dokku ps[:COMMAND] + +Manage app processes + +Additional commands:` + + helpContent = ` + ps:inspect , Displays a sanitized version of docker inspect for an app + ps:scale = [=...], Get/Set how many instances of a given process to run + ps:start [--parallel count] [--serial] [--all|], Start an app + ps:stop [--parallel count] [--serial] [--all|], Stop an app + ps:rebuild [--parallel count] [--serial] [--all|], Rebuilds an app from source + ps:report [] [], Displays a process report for one or more apps + ps:restart [--parallel count] [--serial] [--all|], Restart an app + ps:restore, Start previously running apps e.g. after reboot +` +) + +func main() { + flag.Usage = usage + flag.Parse() + + cmd := flag.Arg(0) + switch cmd { + case "ps", "ps:help": + usage() + case "help": + command := common.NewShellCmd(fmt.Sprintf("ps -o command= %d", os.Getppid())) + command.ShowOutput = false + output, err := command.Output() + + if err == nil && strings.Contains(string(output), "--all") { + fmt.Println(helpContent) + } else { + fmt.Print("\n ps, Manage app processes\n") + } + default: + dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT")) + if err != nil { + fmt.Println("failed to retrieve DOKKU_NOT_IMPLEMENTED_EXIT environment variable") + dokkuNotImplementExitCode = 10 + } + os.Exit(dokkuNotImplementExitCode) + } +} + +func usage() { + common.CommandUsage(helpHeader, helpContent) +} diff --git a/plugins/ps/src/subcommands/subcommands.go b/plugins/ps/src/subcommands/subcommands.go new file mode 100644 index 000000000..cecfc6e75 --- /dev/null +++ b/plugins/ps/src/subcommands/subcommands.go @@ -0,0 +1,92 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/ps" +) + +// main entrypoint to all subcommands +func main() { + parts := strings.Split(os.Args[0], "/") + subcommand := parts[len(parts)-1] + + var err error + switch subcommand { + case "inspect": + args := flag.NewFlagSet("ps:inspect", flag.ExitOnError) + args.Parse(os.Args[2:]) + appName := args.Arg(0) + err = ps.CommandInspect(appName) + case "rebuild": + args := flag.NewFlagSet("ps:rebuild", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: restart all apps") + parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + runInSerial := args.Bool("serial", false, "--serial: restart apps serially") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + err = ps.CommandRebuild(appName, *allApps, *runInSerial, *parallelCount) + case "report": + args := flag.NewFlagSet("ps:report", flag.ExitOnError) + args.Parse(os.Args[2:]) + appName := args.Arg(0) + infoFlag := args.Arg(1) + err = ps.CommandReport(appName, infoFlag) + case "restart": + args := flag.NewFlagSet("ps:restart", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: restart all apps") + parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + runInSerial := args.Bool("serial", false, "--serial: restart apps serially") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + err = ps.CommandRestart(appName, *allApps, *runInSerial, *parallelCount) + case "restore": + args := flag.NewFlagSet("ps:restore", flag.ExitOnError) + args.Parse(os.Args[2:]) + appName := args.Arg(0) + err = ps.CommandRestore(appName) + case "retire": + args := flag.NewFlagSet("ps:retire", flag.ExitOnError) + args.Parse(os.Args[2:]) + err = ps.CommandRetire() + case "scale": + args := flag.NewFlagSet("ps:scale", flag.ExitOnError) + args.Parse(os.Args[2:]) + appName := args.Arg(0) + _, processTuples := common.ShiftString(args.Args()) + err = ps.CommandScale(appName, processTuples) + case "set": + args := flag.NewFlagSet("ps:set", flag.ExitOnError) + args.Parse(os.Args[2:]) + appName := args.Arg(0) + property := args.Arg(1) + value := args.Arg(2) + err = ps.CommandSet(appName, property, value) + case "start": + args := flag.NewFlagSet("ps:start", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: restart all apps") + parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + runInSerial := args.Bool("serial", false, "--serial: restart apps serially") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + err = ps.CommandStart(appName, *allApps, *runInSerial, *parallelCount) + case "stop": + args := flag.NewFlagSet("ps:stop", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: restart all apps") + parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + runInSerial := args.Bool("serial", false, "--serial: restart apps serially") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + err = ps.CommandStop(appName, *allApps, *runInSerial, *parallelCount) + default: + common.LogFail(fmt.Sprintf("Invalid plugin subcommand call: %s", subcommand)) + } + + if err != nil { + common.LogFail(err.Error()) + } +} diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go new file mode 100644 index 000000000..3ded508cb --- /dev/null +++ b/plugins/ps/src/triggers/triggers.go @@ -0,0 +1,80 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/ps" +) + +// main entrypoint to all triggers +func main() { + parts := strings.Split(os.Args[0], "/") + trigger := parts[len(parts)-1] + flag.Parse() + + var err error + switch trigger { + case "app-restart": + appName := flag.Arg(0) + err = ps.TriggerAppRestart(appName) + case "core-post-deploy": + appName := flag.Arg(0) + err = ps.TriggerCorePostDeploy(appName) + case "install": + err = ps.TriggerInstall() + case "post-app-clone": + oldAppName := flag.Arg(0) + newAppName := flag.Arg(1) + err = ps.TriggerPostAppClone(oldAppName, newAppName) + case "post-app-rename": + oldAppName := flag.Arg(0) + newAppName := flag.Arg(1) + err = ps.TriggerPostAppRename(oldAppName, newAppName) + case "post-create": + appName := flag.Arg(0) + err = ps.TriggerPostCreate(appName) + case "post-delete": + appName := flag.Arg(0) + err = ps.TriggerPostDelete(appName) + case "post-extract": + appName := flag.Arg(0) + tmpWorkDir := flag.Arg(1) + err = ps.TriggerPostExtract(appName, tmpWorkDir) + case "post-stop": + appName := flag.Arg(0) + err = ps.TriggerPostStop(appName) + case "pre-deploy": + appName := flag.Arg(0) + imageTag := flag.Arg(1) + err = ps.TriggerPreDeploy(appName, imageTag) + case "procfile-extract": + appName := flag.Arg(0) + image := flag.Arg(1) + forceExtract := common.ToBool(flag.Arg(2)) + procfilePath := flag.Arg(3) + err = ps.TriggerProcfileExtract(appName, image, forceExtract, procfilePath) + case "procfile-get-command": + appName := flag.Arg(0) + processType := flag.Arg(1) + port := common.ToInt(flag.Arg(2), 5000) + procfilePath := flag.Arg(3) + err = ps.TriggerProcfileGetCommand(appName, processType, port, procfilePath) + case "procfile-remove": + appName := flag.Arg(0) + procfilePath := flag.Arg(1) + err = ps.TriggerProcfileRemove(appName, procfilePath) + case "report": + appName := flag.Arg(0) + err = ps.ReportSingleApp(appName, "") + default: + common.LogFail(fmt.Sprintf("Invalid plugin trigger call: %s", trigger)) + } + + if err != nil { + common.LogFail(err.Error()) + } +} diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go new file mode 100644 index 000000000..21a27739a --- /dev/null +++ b/plugins/ps/subcommands.go @@ -0,0 +1,215 @@ +package ps + +import ( + "errors" + "fmt" + "os" + "sort" + "strings" + + dockeroptions "github.com/dokku/dokku/plugins/docker-options" + + "github.com/dokku/dokku/plugins/common" + "github.com/ryanuber/columnize" +) + +func CommandInspect(appName string) error { + if appName == "" { + return errors.New("Please specify an app to run the command on") + } + + if err := common.VerifyAppName(appName); err != nil { + return err + } + + scheduler := common.GetAppScheduler(appName) + return common.PlugnTrigger("scheduler-inspect", []string{scheduler, appName}...) +} + +func CommandRebuild(appName string, allApps bool, runInSerial bool, parallelCount int) error { + if allApps { + return RunCommandAgainstAllApps(Rebuild, "rebuild", runInSerial, parallelCount) + } + + if appName == "" { + return errors.New("Please specify an app to run the command on") + } + + if err := common.VerifyAppName(appName); err != nil { + return err + } + + return Rebuild(appName) +} + +// CommandReport displays a ps report for one or more apps +func CommandReport(appName string, infoFlag string) error { + if strings.HasPrefix(appName, "--") { + infoFlag = appName + appName = "" + } + + if len(appName) == 0 { + apps, err := common.DokkuApps() + if err != nil { + return err + } + for _, appName := range apps { + if err := ReportSingleApp(appName, infoFlag); err != nil { + return err + } + } + return nil + } + + return ReportSingleApp(appName, infoFlag) +} + +func CommandRestart(appName string, allApps bool, runInSerial bool, parallelCount int) error { + if allApps { + return RunCommandAgainstAllApps(Restart, "restart", runInSerial, parallelCount) + } + + if appName == "" { + return errors.New("Please specify an app to run the command on") + } + + if err := common.VerifyAppName(appName); err != nil { + return err + } + + return Restart(appName) +} + +// TODO: implement me +func CommandRestore(appName string) error { + return nil +} + +// TODO: implement me +func CommandRetire() error { + return nil +} + +// TODO: implement me +func CommandScale(appName string, processTuples []string) error { + if appName == "" { + return errors.New("Please specify an app to run the command on") + } + + if err := common.VerifyAppName(appName); err != nil { + return err + } + + procfilePath := getProcfilePath(appName) + scalefilePath := getScalefilePath(appName) + if !common.FileExists(procfilePath) { + image := common.GetAppImageRepo(appName) + extractProcfile(appName, image, procfilePath) + } + + if !common.FileExists(scalefilePath) { + err := common.SuppressOutput(func() error { + return generateScalefile(appName, scalefilePath) + }) + if err != nil { + return err + } + } else if common.FileExists(procfilePath) { + if err := updateScalefile(appName, []string{}); err != nil { + return err + } + } + + if len(processTuples) == 0 { + lines, err := common.FileToSlice(scalefilePath) + if err != nil { + return err + } + + common.LogInfo1Quiet(fmt.Sprintf("Scaling for %s", appName)) + config := columnize.DefaultConfig() + config.Delim = "=" + config.Glue = ": " + config.Prefix = " " + config.Empty = "" + + content := []string{} + if os.Getenv("DOKKU_QUIET_OUTPUT") == "" { + content = append(content, "proctype=qty","--------=---") + } + + sort.Strings(lines) + for _, line := range lines { + content = append(content, line) + } + + for _, line := range content { + s := strings.Split(line, "=") + common.Log(fmt.Sprintf("%s %s", common.RightPad(fmt.Sprintf("%s:", s[0]), 5, " "), s[1])) + } + } else { + if !canScaleApp(appName) { + return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) + } + + if err := updateScalefile(appName, processTuples); err != nil { + return err + } + + imageTag, err := common.GetRunningImageTag(appName) + if err != nil { + return err + } + return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) + } + + return nil +} + +// TODO: implement me +func CommandSet(appName string, property string, value string) error { + if property == "restart-policy" { + if !isValidRestartPolicy(value) { + return errors.New("Invalid restart-policy specified") + } + + common.LogInfo2Quiet(fmt.Sprintf("Setting %s to %s", property, value)) + return dockeroptions.SetDockerOptionForPhases(appName, []string{"deploy"}, "restart", value) + } + + common.CommandPropertySet("ps", appName, property, value, DefaultProperties) + return nil +} + +func CommandStart(appName string, allApps bool, runInSerial bool, parallelCount int) error { + if allApps { + return RunCommandAgainstAllApps(Start, "start", runInSerial, parallelCount) + } + + if appName == "" { + return errors.New("Please specify an app to run the command on") + } + + if err := common.VerifyAppName(appName); err != nil { + return err + } + + return Start(appName) +} + +func CommandStop(appName string, allApps bool, runInSerial bool, parallelCount int) error { + if allApps { + return RunCommandAgainstAllApps(Stop, "stop", runInSerial, parallelCount) + } + + if appName == "" { + return errors.New("Please specify an app to run the command on") + } + + if err := common.VerifyAppName(appName); err != nil { + return err + } + + return Stop(appName) +} diff --git a/plugins/ps/subcommands/default b/plugins/ps/subcommands/default deleted file mode 100755 index 55742f338..000000000 --- a/plugins/ps/subcommands/default +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/ps/help-functions" - -cmd-ps-help "ps:help" diff --git a/plugins/ps/subcommands/inspect b/plugins/ps/subcommands/inspect deleted file mode 100755 index acea3e2b9..000000000 --- a/plugins/ps/subcommands/inspect +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x - -cmd-ps-inspect "$@" diff --git a/plugins/ps/subcommands/rebuild b/plugins/ps/subcommands/rebuild deleted file mode 100755 index 22d1ba2dd..000000000 --- a/plugins/ps/subcommands/rebuild +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -cmd-ps-rebuild() { - declare desc="rebuilds an app from source" - declare cmd="ps:rebuild" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - - [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" - - if (is_deployed "$APP"); then - ps_rebuild "$APP" - else - dokku_log_warn "App $APP has not been deployed" - fi -} - -cmd-ps-rebuild "$@" diff --git a/plugins/ps/subcommands/rebuildall b/plugins/ps/subcommands/rebuildall deleted file mode 100755 index 98fe32095..000000000 --- a/plugins/ps/subcommands/rebuildall +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" - -cmd-ps-rebuildall() { - declare desc="rebuilds all apps from source" - declare cmd="ps:rebuildall" - [[ "$1" == "$cmd" ]] && shift 1 - - fn-ps-parallel-cmd "rebuild" -} - -cmd-ps-rebuildall "$@" diff --git a/plugins/ps/subcommands/report b/plugins/ps/subcommands/report deleted file mode 100755 index 0a967fd0d..000000000 --- a/plugins/ps/subcommands/report +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x - -cmd-ps-report "$@" diff --git a/plugins/ps/subcommands/restart b/plugins/ps/subcommands/restart deleted file mode 100755 index 582fe82ca..000000000 --- a/plugins/ps/subcommands/restart +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" - -cmd-ps-restart() { - declare desc="restarts app via command line" - declare cmd="ps:restart" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - - [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" - plugn trigger app-restart "$APP" -} - -cmd-ps-restart "$@" diff --git a/plugins/ps/subcommands/restart-policy b/plugins/ps/subcommands/restart-policy deleted file mode 100755 index fe30684c5..000000000 --- a/plugins/ps/subcommands/restart-policy +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/docker-options/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -cmd-ps-restart-policy() { - declare desc="shows the restart-policy for an app" - declare cmd="ps:restart-policy" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - local passed_phases="deploy" - - [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" - verify_app_name "$APP" - - dokku_log_info2_quiet "$APP restart-policy:" - get_restart_policies "$(get_phase_file_path "$passed_phases")" -} - -cmd-ps-restart-policy "$@" diff --git a/plugins/ps/subcommands/restartall b/plugins/ps/subcommands/restartall deleted file mode 100755 index cb9200e10..000000000 --- a/plugins/ps/subcommands/restartall +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" - -cmd-ps-restartall() { - declare desc="restarts all apps via command line" - declare cmd="ps:restartall" - [[ "$1" == "$cmd" ]] && shift 1 - - fn-ps-parallel-cmd "restart" -} - -cmd-ps-restartall "$@" diff --git a/plugins/ps/subcommands/restore b/plugins/ps/subcommands/restore deleted file mode 100755 index d3c25df41..000000000 --- a/plugins/ps/subcommands/restore +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" - -cmd-ps-restore() { - declare desc="starts all apps with DOKKU_APP_RESTORE not set to 0 via command line" - declare cmd="ps:restore" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - local DOKKU_SCHEDULER - - DOKKU_SCHEDULER=$(get_app_scheduler "$APP") - plugn trigger pre-restore "$DOKKU_SCHEDULER" "$APP" - - if [[ -n "$APP" ]]; then - plugn trigger proxy-clear-config "$APP" - - if ! (is_deployed "$APP"); then - dokku_log_warn "App $APP has not been deployed" - return - fi - - DOKKU_APP_RESTORE="$(config_get "$APP" DOKKU_APP_RESTORE || true)" - if [[ $DOKKU_APP_RESTORE != 0 ]]; then - ps_restore "$APP" || dokku_log_warn "dokku ps:restore ${APP} failed" - fi - else - for APP in $(dokku_apps); do - plugn trigger proxy-clear-config "$APP" - done - - fn-ps-parallel-cmd "restore" - fi -} - -cmd-ps-restore "$@" diff --git a/plugins/ps/subcommands/retire b/plugins/ps/subcommands/retire deleted file mode 100755 index d32e6b8af..000000000 --- a/plugins/ps/subcommands/retire +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" - -cmd-ps-retire() { - declare desc="ensures old containers are retired" - declare cmd="ps:retire" - [[ "$1" == "$cmd" ]] && shift 1 - - local LOCK_FILE="${DOKKU_LIB_ROOT}/data/ps/retire" - local DOKKU_SCHEDULER=$(config_get "--global" DOKKU_SCHEDULER || echo "docker-local") - - acquire_advisory_lock "$LOCK_FILE" "exclusive" "" "Failed to acquire ps:retire lock" - plugn trigger scheduler-retire "$DOKKU_SCHEDULER" - release_advisory_lock "$LOCK_FILE" -} - -cmd-ps-retire "$@" diff --git a/plugins/ps/subcommands/scale b/plugins/ps/subcommands/scale deleted file mode 100755 index 5a5a6e4bc..000000000 --- a/plugins/ps/subcommands/scale +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -cmd-ps-scale() { - declare desc="sets app scaling config via command line" - declare cmd="ps:scale" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - - [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" - ps_scale "$@" -} - -cmd-ps-scale "$@" diff --git a/plugins/ps/subcommands/set-restart-policy b/plugins/ps/subcommands/set-restart-policy deleted file mode 100755 index b18495e2e..000000000 --- a/plugins/ps/subcommands/set-restart-policy +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/docker-options/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -cmd-ps-set-restart-policy() { - declare desc="sets app restart-policy" - declare cmd="ps:set-restart-policy" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" RESTART_POLICY="$2" - - [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" - verify_app_name "$APP" - local RE_RESTART_POLICY="^(no|unless-stopped|always|on-failure(:[1-9][0-9]*)?)$" - - if [[ ! "$RESTART_POLICY" =~ $RE_RESTART_POLICY ]]; then - dokku_log_fail "Please specify an valid restart policy matching the following regex: ${RE_RESTART_POLICY}" - fi - - local RESTART_POLICIES=$(get_raw_restart_policies "$(get_phase_file_path "deploy")") - local passed_phases=(deploy) - - for policy in $RESTART_POLICIES; do - remove_passed_docker_option passed_phases[@] "$policy" - done - - dokku_log_info1_quiet "Setting restart policy: ${RESTART_POLICY}" - local passed_phases=(deploy) - add_passed_docker_option passed_phases[@] "--restart=${RESTART_POLICY}" -} - -cmd-ps-set-restart-policy "$@" diff --git a/plugins/ps/subcommands/start b/plugins/ps/subcommands/start deleted file mode 100755 index 8be6cae51..000000000 --- a/plugins/ps/subcommands/start +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -cmd-ps-start() { - declare desc="starts app via command line" - declare cmd="ps:start" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - - [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" - ps_start "$APP" -} - -cmd-ps-start "$@" diff --git a/plugins/ps/subcommands/startall b/plugins/ps/subcommands/startall deleted file mode 100755 index 69726033c..000000000 --- a/plugins/ps/subcommands/startall +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" - -cmd-ps-startall() { - declare desc="starts all apps via command line" - declare cmd="ps:startall" - [[ "$1" == "$cmd" ]] && shift 1 - - fn-ps-parallel-cmd "start" -} - -cmd-ps-startall "$@" diff --git a/plugins/ps/subcommands/stop b/plugins/ps/subcommands/stop deleted file mode 100755 index 37c0fec66..000000000 --- a/plugins/ps/subcommands/stop +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -cmd-ps-stop() { - declare desc="stops app via command line" - declare cmd="ps:stop" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - - [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" - ps_stop "$APP" -} - -cmd-ps-stop "$@" diff --git a/plugins/ps/subcommands/stopall b/plugins/ps/subcommands/stopall deleted file mode 100755 index f05be6a45..000000000 --- a/plugins/ps/subcommands/stopall +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/internal-functions" - -cmd-ps-stopall() { - declare desc="stops all apps via command line" - declare cmd="ps:stopall" - [[ "$1" == "$cmd" ]] && shift 1 - - fn-ps-parallel-cmd "stop" -} - -cmd-ps-stopall "$@" diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go new file mode 100644 index 000000000..201ea22ef --- /dev/null +++ b/plugins/ps/triggers.go @@ -0,0 +1,193 @@ +package ps + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config" + dockeroptions "github.com/dokku/dokku/plugins/docker-options" + sh "github.com/codeskyblue/go-sh" +) + +func TriggerAppRestart(appName string) error { + return Restart(appName) +} + +func TriggerCorePostDeploy(appName string) error { + if err := removeProcfile(appName); err != nil { + return err + } + + entries := map[string]string{ + "DOKKU_APP_RESTORE": "1", + } + + return common.SuppressOutput(func() error { + return config.SetMany(appName, entries, false) + }) +} + +func TriggerInstall() error { + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps") + if err := os.MkdirAll(directory, 0755); err != nil { + return err + } + + if err := common.SetPermissions(directory, 0755); err != nil { + return err + } + + apps, err := common.DokkuApps() + if err != nil { + return nil + } + + for _, appName := range apps { + policies, err := getRestartPolicy(appName) + if err != nil { + return err + } + + if len(policies) != 0 { + continue + } + + if err := dockeroptions.AddDockerOptionToPhases(appName, []string{"deploy"}, "--restart=on-failure:10"); err != nil { + common.LogWarn(err.Error()) + } + } + + return nil +} + +func TriggerPostAppClone(oldAppName string, newAppName string) error { + if os.Getenv("SKIP_REBUILD") == "true" { + return nil + } + + return Rebuild(newAppName) +} + +func TriggerPostAppRename(oldAppName string, newAppName string) error { + if os.Getenv("SKIP_REBUILD") == "true" { + return nil + } + + return Rebuild(newAppName) +} + +func TriggerPostCreate(appName string) error { + if err := dockeroptions.AddDockerOptionToPhases(appName, []string{"deploy"}, "--restart=on-failure:10"); err != nil { + return err + } + + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", appName) + if err := os.MkdirAll(directory, 0755); err != nil { + return err + } + + if err := common.SetPermissions(directory, 0755); err != nil { + return err + } + + return nil +} + +// TriggerPostDelete destroys the ps properties for a given app container +func TriggerPostDelete(appName string) error { + return common.PropertyDestroy("ps", appName) +} + +func TriggerPostExtract(appName string, tempWorkDir string) error { + procfile := filepath.Join(tempWorkDir, "Procfile") + if !common.FileExists(procfile) { + return nil + } + + b, err := sh.Command("procfile-util", "check", "-P", procfile).CombinedOutput() + if err != nil { + return fmt.Errorf(strings.TrimSpace(string(b[:]))) + } + return nil +} + +func TriggerPostStop(appName string) error { + entries := map[string]string{ + "DOKKU_APP_RESTORE": "0", + } + + return common.SuppressOutput(func() error { + return config.SetMany(appName, entries, false) + }) +} + +func TriggerPreDeploy(appName string, imageTag string) error { + image := common.GetAppImageRepo(appName) + removeProcfile(appName) + + procfilePath := getProcfilePath(appName) + if err := extractProcfile(appName, image, procfilePath); err != nil { + return err + } + if err := extractOrGenerateScalefile(appName, imageTag); err != nil { + return err + } + + return nil +} + +func TriggerProcfileExtract(appName string, image string, forceExtract bool, procfilePath string) error { + if procfilePath == "" { + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", appName) + if err := os.MkdirAll(directory, 0755); err != nil { + return err + } + + if err := common.SetPermissions(directory, 0755); err != nil { + return err + } + + procfilePath = getProcfilePath(appName) + } + + if forceExtract { + if err := common.PlugnTrigger("procfile-remove", []string{appName, procfilePath}...); err != nil { + return err + } + } + + return extractProcfile(appName, image, procfilePath) +} + +func TriggerProcfileGetCommand(appName string, processType string, port int, procfilePath string) error { + if procfilePath == "" { + procfilePath = getProcfilePath(appName) + } + + command, err := getProcfileCommand(procfilePath, processType, port) + if err != nil { + return err + } + + if command != "" { + fmt.Printf("%s\n", command) + } + + return nil +} + +func TriggerProcfileRemove(appName string, procfilePath string) error { + if procfilePath == "" { + procfilePath = getProcfilePath(appName) + } + + if !common.FileExists(procfilePath) { + return nil + } + + os.Remove(procfilePath) + return nil +} diff --git a/tests/unit/ps-general.bats b/tests/unit/ps-general.bats index 4b2fc7220..99bc2bea6 100644 --- a/tests/unit/ps-general.bats +++ b/tests/unit/ps-general.bats @@ -16,13 +16,13 @@ teardown() { run /bin/bash -c "dokku ps" echo "output: $output" echo "status: $status" - assert_output_contains "Manage container processes" + assert_output_contains "Manage app processes" help_output="$output" run /bin/bash -c "dokku ps:help" echo "output: $output" echo "status: $status" - assert_output_contains "Manage container processes" + assert_output_contains "Manage app processes" assert_output "$help_output" } @@ -64,7 +64,7 @@ EOF run /bin/bash -c "dokku --quiet ps:scale $TEST_APP" output=$(echo "$output" | tr -s " ") echo "output: ($output)" - assert_output $'cron: 0 \ncustom: 0 \nrelease: 0 \nweb: 1 \nworker: 0 ' + assert_output $'cron: 0\ncustom: 0\nrelease: 0\nweb: 1\nworker: 0' pushd $TMP echo scaletest: sleep infinity >> Procfile @@ -74,28 +74,27 @@ EOF run /bin/bash -c "dokku --quiet ps:scale $TEST_APP" output=$(echo "$output" | tr -s " ") echo "output: ($output)" - assert_output $'cron: 0 \ncustom: 0 \nrelease: 0 \nweb: 1 \nworker: 0 \nscaletest: 0 ' + assert_output $'cron: 0\ncustom: 0\nrelease: 0\nscaletest: 0\nweb: 1\nworker: 0' popd rm -rf "$TMP" } @test "(ps:restart-policy) default policy" { - run /bin/bash -c "dokku --quiet ps:restart-policy $TEST_APP" + run /bin/bash -c "dokku --quiet ps:report $TEST_APP --ps-restart-policy" echo "output: $output" echo "status: $status" assert_output "on-failure:10" } - -@test "(ps:restart-policy) ps:set-restart-policy, ps:restart-policy" { +@test "(ps:restart-policy) ps:set restart-policy" { for policy in no unless-stopped always on-failure on-failure:20; do - run /bin/bash -c "dokku ps:set-restart-policy $TEST_APP $policy" + run /bin/bash -c "dokku ps:set $TEST_APP restart-policy $policy" echo "output: $output" echo "status: $status" assert_success - run /bin/bash -c "dokku --quiet ps:restart-policy $TEST_APP" + run /bin/bash -c "dokku --quiet ps:report $TEST_APP --ps-restart-policy" echo "output: $output" echo "status: $status" assert_output "$policy" @@ -104,12 +103,12 @@ EOF @test "(ps:restart-policy) deployed policy" { test_restart_policy="on-failure:20" - run /bin/bash -c "dokku ps:set-restart-policy $TEST_APP $test_restart_policy" + run /bin/bash -c "dokku ps:set $TEST_APP restart-policy $test_restart_policy" echo "output: $output" echo "status: $status" assert_success - run /bin/bash -c "dokku --quiet ps:restart-policy $TEST_APP" + run /bin/bash -c "dokku --quiet ps:report $TEST_APP --ps-restart-policy" echo "output: $output" echo "status: $status" assert_output "$test_restart_policy" From 70de587f6cc188b712fa983032e66f061f246e9d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 15:54:57 -0500 Subject: [PATCH 18/67] feat: upgrade procfile-util to avoid errors in parsing --- Makefile | 2 +- debian/control | 2 +- rpm.mk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5e17c1bca..b11005ca0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ DOKKU_VERSION ?= master HEROKUISH_VERSION ?= 0.5.19 -PROCFILE_VERSION ?= 0.8.2 +PROCFILE_VERSION ?= 0.11.0 PLUGN_VERSION ?= 0.5.0 SIGIL_VERSION ?= 0.6.0 SSHCOMMAND_VERSION ?= 0.11.0 diff --git a/debian/control b/debian/control index 6945e953b..d6d6ad70b 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.21.4 Section: web Priority: optional Architecture: amd64 -Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.10.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, net-tools, software-properties-common, procfile-util (>= 0.9.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq +Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.10.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, net-tools, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq Recommends: herokuish (>= 0.3.4), parallel, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/rpm.mk b/rpm.mk index feab42f8b..a2aa7f1d5 100644 --- a/rpm.mk +++ b/rpm.mk @@ -42,7 +42,7 @@ endif --depends 'nc' \ --depends 'nginx >= 1.8.0' \ --depends 'plugn' \ - --depends 'procfile-util >= 0.9.0' \ + --depends 'procfile-util >= 0.11.0' \ --depends '/usr/bin/python3' \ --depends 'sshcommand >= 0.10.0' \ --depends 'sudo' \ From e8cfbfad36e362133b431dc366650a1f1c15b726 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 16:00:32 -0500 Subject: [PATCH 19/67] fix: drop todos --- plugins/ps/ps.go | 1 - plugins/ps/subcommands.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index a4e7b1db9..201fcad88 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -61,7 +61,6 @@ func Rebuild(appName string) error { } // ReportSingleApp is an internal function that displays the ps report for one or more apps -// TODO: implement me func ReportSingleApp(appName, infoFlag string) error { if err := common.VerifyAppName(appName); err != nil { return err diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 21a27739a..352e3ae78 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -91,7 +91,6 @@ func CommandRetire() error { return nil } -// TODO: implement me func CommandScale(appName string, processTuples []string) error { if appName == "" { return errors.New("Please specify an app to run the command on") @@ -167,7 +166,6 @@ func CommandScale(appName string, processTuples []string) error { return nil } -// TODO: implement me func CommandSet(appName string, property string, value string) error { if property == "restart-policy" { if !isValidRestartPolicy(value) { From ab1d15216c14fe67e5796d95f65b6da3cedcbba7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 17:30:24 -0500 Subject: [PATCH 20/67] refactor: drop the last argument from procfile-get-command No external plugin should need to know the internals of where processes are. --- docs/appendices/0.22.0-migration-guide.md | 1 + docs/development/plugin-triggers.md | 2 +- plugins/ps/src/triggers/triggers.go | 2 +- plugins/ps/triggers.go | 7 ++----- tests/unit/ps-general.bats | 4 ++-- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/appendices/0.22.0-migration-guide.md b/docs/appendices/0.22.0-migration-guide.md index ca5b96cbe..f520a56de 100644 --- a/docs/appendices/0.22.0-migration-guide.md +++ b/docs/appendices/0.22.0-migration-guide.md @@ -17,3 +17,4 @@ The `ps` command has had a few removals as a result of a rewrite to Golang: - The `ps:stopall` command has been removed in favor of calling the `ps:stop` with the `--all` flag instead of an app name. - The `ps:startall` command has been removed in favor of calling the `ps:start` with the `--all` flag instead of an app name. - The `DOKKU_PROCFILE` file is no longer located in the git directory for a given app on the server. It's location is currently not exposed. +- The `procfile-get-command` plugin trigger no longer takes a `procfile path` as the final argument. diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 92c356654..fd3c6fb10 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1496,7 +1496,7 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x - Description: Fetches the command for a specific process type - Invoked by: `internally` -- Arguments: `$APP $PROCESS_TYPE $PORT $PROCFILE_PATH` +- Arguments: `$APP $PROCESS_TYPE $PORT` - Example: ```shell diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index 3ded508cb..6618178b0 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -62,7 +62,7 @@ func main() { processType := flag.Arg(1) port := common.ToInt(flag.Arg(2), 5000) procfilePath := flag.Arg(3) - err = ps.TriggerProcfileGetCommand(appName, processType, port, procfilePath) + err = ps.TriggerProcfileGetCommand(appName, processType, port) case "procfile-remove": appName := flag.Arg(0) procfilePath := flag.Arg(1) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 201ea22ef..656d2d603 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -162,11 +162,8 @@ func TriggerProcfileExtract(appName string, image string, forceExtract bool, pro return extractProcfile(appName, image, procfilePath) } -func TriggerProcfileGetCommand(appName string, processType string, port int, procfilePath string) error { - if procfilePath == "" { - procfilePath = getProcfilePath(appName) - } - +func TriggerProcfileGetCommand(appName string, processType string, port int) error { + procfilePath := getProcfilePath(appName) command, err := getProcfileCommand(procfilePath, processType, port) if err != nil { return err diff --git a/tests/unit/ps-general.bats b/tests/unit/ps-general.bats index 99bc2bea6..534966d42 100644 --- a/tests/unit/ps-general.bats +++ b/tests/unit/ps-general.bats @@ -44,12 +44,12 @@ teardown() { web: node web.js --port \$PORT worker: node worker.js EOF - run get_cmd_from_procfile "$TEST_APP" web 5001 + run plugn trigger procfile-get-command "$TEST_APP" web 5001 echo "output: $output" echo "status: $status" assert_output "node web.js --port 5001" - run get_cmd_from_procfile "$TEST_APP" worker + run plugn trigger procfile-get-command "$TEST_APP" worker echo "output: $output" echo "status: $status" assert_output "node worker.js" From 06ce3dfadbf3733fec639cb60e1cc12ab25617e1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 17:39:07 -0500 Subject: [PATCH 21/67] fix: remove unused variable --- plugins/ps/src/triggers/triggers.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index 6618178b0..bf3eac8a8 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -61,7 +61,6 @@ func main() { appName := flag.Arg(0) processType := flag.Arg(1) port := common.ToInt(flag.Arg(2), 5000) - procfilePath := flag.Arg(3) err = ps.TriggerProcfileGetCommand(appName, processType, port) case "procfile-remove": appName := flag.Arg(0) From f9969a0e61f07f1a2c31344d001939ae19b5c07f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 17:40:26 -0500 Subject: [PATCH 22/67] refactor: drop extra args from procfile-extract They were not necessary. --- docs/development/plugin-triggers.md | 2 +- plugins/app-json/appjson.go | 4 +--- plugins/ps/src/triggers/triggers.go | 4 +--- plugins/ps/triggers.go | 24 +++++++++++------------- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index fd3c6fb10..5a278a6b7 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1481,7 +1481,7 @@ curl "https://dokku.me/starting/${APP}" || true - Description: Extracts a Procfile from a given image to a path - Invoked by: `internally` -- Arguments: `$APP $IMAGE $FORCE_EXTRACT $PROCFILE_PATH` +- Arguments: `$APP $IMAGE` - Example: ```shell diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 05057cffa..bcb077ad4 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -110,10 +110,8 @@ func getPhaseScript(appName string, image string, phase string) (string, error) // getReleaseCommand extracts the release command from a given app's procfile func getReleaseCommand(appName string, image string) string { - forceExtract := "true" - err := common.SuppressOutput(func() error { - return common.PlugnTrigger("procfile-extract", []string{appName, image, forceExtract}...) + return common.PlugnTrigger("procfile-extract", []string{appName, image}...) }) if err != nil { diff --git a/plugins/ps/src/triggers/triggers.go b/plugins/ps/src/triggers/triggers.go index bf3eac8a8..d58084abe 100644 --- a/plugins/ps/src/triggers/triggers.go +++ b/plugins/ps/src/triggers/triggers.go @@ -54,9 +54,7 @@ func main() { case "procfile-extract": appName := flag.Arg(0) image := flag.Arg(1) - forceExtract := common.ToBool(flag.Arg(2)) - procfilePath := flag.Arg(3) - err = ps.TriggerProcfileExtract(appName, image, forceExtract, procfilePath) + err = ps.TriggerProcfileExtract(appName, image) case "procfile-get-command": appName := flag.Arg(0) processType := flag.Arg(1) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 656d2d603..1b8da30a1 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -139,21 +139,19 @@ func TriggerPreDeploy(appName string, imageTag string) error { return nil } -func TriggerProcfileExtract(appName string, image string, forceExtract bool, procfilePath string) error { - if procfilePath == "" { - directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", appName) - if err := os.MkdirAll(directory, 0755); err != nil { - return err - } - - if err := common.SetPermissions(directory, 0755); err != nil { - return err - } - - procfilePath = getProcfilePath(appName) +func TriggerProcfileExtract(appName string, image string) error { + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", appName) + if err := os.MkdirAll(directory, 0755); err != nil { + return err } - if forceExtract { + if err := common.SetPermissions(directory, 0755); err != nil { + return err + } + + procfilePath := getProcfilePath(appName) + + if common.FileExists(procfilePath) { if err := common.PlugnTrigger("procfile-remove", []string{appName, procfilePath}...); err != nil { return err } From cae561f7b994c0adbc433dcf516e350f1e05dee2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 17:42:09 -0500 Subject: [PATCH 23/67] feat: extract procfile when fetching command as necessary --- plugins/ps/triggers.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 1b8da30a1..63b85e6bd 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -162,6 +162,12 @@ func TriggerProcfileExtract(appName string, image string) error { func TriggerProcfileGetCommand(appName string, processType string, port int) error { procfilePath := getProcfilePath(appName) + if !common.FileExists(procfilePath) { + image := common.GetDeployingAppImageName(appName, "", "") + if err := common.PlugnTrigger("procfile-extract", []string{appName, image}...); err != nil { + return err + } + } command, err := getProcfileCommand(procfilePath, processType, port) if err != nil { return err From a93f504d4548a26ea86761cb9a6ecdb66afa328f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 17:42:32 -0500 Subject: [PATCH 24/67] chore: short-circuit procfile command fetching if there is no procfile --- plugins/ps/functions.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 45fcfa151..a4ddd52fa 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -2,6 +2,7 @@ package ps import ( "bytes" + "errors" "fmt" "os" "path/filepath" @@ -101,6 +102,10 @@ func generateScalefile(appName string, destination string) error { } func getProcfileCommand(procfilePath string, processType string, port int) (string, error) { + if !common.FileExists(procfilePath) { + return "", errors.New("No procfile found") + } + shellCmd := common.NewShellCmd(strings.Join([]string{ "procfile-util", "show", From ebdf38e136b8155d3b2adf07c2c17bab5e933537 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 17:44:41 -0500 Subject: [PATCH 25/67] refactor: use procfile-get-command plugin trigger instead of direct ps function calls --- plugins/scheduler-docker-local/scheduler-deploy | 3 +-- plugins/scheduler-docker-local/scheduler-run | 7 +------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index a987d7f66..24be945b2 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -4,7 +4,6 @@ set -eo pipefail source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/checks/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions" trigger-scheduler-docker-local-scheduler-deploy() { @@ -217,7 +216,7 @@ fn-scheduler-docker-local-extract-start-cmd() { # TODO: remove direct procfile access DOKKU_DOCKERFILE_START_CMD=$(config_get "$APP" DOKKU_DOCKERFILE_START_CMD || true) - DOKKU_PROCFILE_START_CMD=$(get_cmd_from_procfile "$APP" "$PROC_TYPE" "$PORT") + DOKKU_PROCFILE_START_CMD=$(plugn trigger procfile-get-command "$APP" "$PROC_TYPE" "$PORT") START_CMD=${DOKKU_DOCKERFILE_START_CMD:-$DOKKU_PROCFILE_START_CMD} echo "$START_CMD" } diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index fe2e2f299..441165ace 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -3,7 +3,6 @@ set -eo pipefail [[ $DOKKU_TRACE ]] && set -x source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" trigger-scheduler-docker-local-scheduler-run() { declare desc="runs command in container based on app image" @@ -63,12 +62,8 @@ trigger-scheduler-docker-local-scheduler-run() { [[ "$IMAGE_SOURCE_TYPE" == "herokuish" ]] && local EXEC_CMD="/exec" - # TODO: remove direct procfile access - DOKKU_QUIET_OUTPUT=1 extract_procfile "$APP" - POTENTIAL_PROCFILE_KEY="$1" - PROC_CMD=$(get_cmd_from_procfile "$APP" "$POTENTIAL_PROCFILE_KEY" 2>/dev/null || echo '') - remove_procfile "$APP" + PROC_CMD=$(plugn trigger procfile-get-comand "$APP" "$POTENTIAL_PROCFILE_KEY" "5000" 2>/dev/null || echo '') if [[ -n "$PROC_CMD" ]]; then dokku_log_info1_quiet "Found '$POTENTIAL_PROCFILE_KEY' in Procfile, running that command" From 815cd530c8a834893d194c7bf53a419eaf290e3e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 17:45:44 -0500 Subject: [PATCH 26/67] chore: remove functions --- docs/appendices/0.22.0-migration-guide.md | 1 + plugins/ps/functions | 255 ---------------------- plugins/ps/internal-functions | 115 ---------- 3 files changed, 1 insertion(+), 370 deletions(-) diff --git a/docs/appendices/0.22.0-migration-guide.md b/docs/appendices/0.22.0-migration-guide.md index f520a56de..a4e4dc7f4 100644 --- a/docs/appendices/0.22.0-migration-guide.md +++ b/docs/appendices/0.22.0-migration-guide.md @@ -18,3 +18,4 @@ The `ps` command has had a few removals as a result of a rewrite to Golang: - The `ps:startall` command has been removed in favor of calling the `ps:start` with the `--all` flag instead of an app name. - The `DOKKU_PROCFILE` file is no longer located in the git directory for a given app on the server. It's location is currently not exposed. - The `procfile-get-command` plugin trigger no longer takes a `procfile path` as the final argument. +- All `ps` functions have been removed. Please us a trigger as appropriate or file a PR to have a new trigger/wrapper function added. diff --git a/plugins/ps/functions b/plugins/ps/functions index fec5beb55..7f5eecd21 100755 --- a/plugins/ps/functions +++ b/plugins/ps/functions @@ -2,135 +2,6 @@ set -eo pipefail [[ $DOKKU_TRACE ]] && set -x source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" - -get_cmd_from_procfile() { - declare desc="parse cmd from app Procfile" - declare APP="$1" PROC_TYPE="$2" PORT="${3:-5000}" - local DOKKU_PROCFILE="$DOKKU_ROOT/$APP/DOKKU_PROCFILE" - local COMMAND - verify_app_name "$APP" - - fn-procfile-get-command "$DOKKU_PROCFILE" "$PROC_TYPE" "$PORT" -} - -print_dokku_scale_file() { - declare desc="prints contents of DOKKU_SCALE file" - local APP="$1" - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" - dokku_log_verbose_quiet "DOKKU_SCALE declares scale -> $(xargs <"$DOKKU_SCALE_FILE")" -} - -extract_procfile() { - declare desc="extracts procfile from app image" - local APP="$1" - local IMAGE_TAG="$2" - local IMAGE="$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")" - local DOKKU_PROCFILE="$DOKKU_ROOT/$APP/DOKKU_PROCFILE" - verify_app_name "$APP" - - remove_procfile "$APP" - fn-procfile-extract "$IMAGE" "$DOKKU_PROCFILE" -} - -fn-procfile-extract() { - declare desc="extract procfile from a specified docker image" - declare IMAGE="$1" PROCFILE_PATH="$2" - - copy_from_image "$IMAGE" "Procfile" "$PROCFILE_PATH" 2>/dev/null || true - if [[ -f "$PROCFILE_PATH" ]]; then - dokku_log_info1_quiet "App Procfile file found" - # shellcheck disable=SC2069 - PROCFILE_ERRORS="$(procfile-util check --procfile "$PROCFILE_PATH" 2>&1 >/dev/null || true)" - if [[ -n "$PROCFILE_ERRORS" ]]; then - dokku_log_fail "${PROCFILE_ERRORS}" - fi - else - dokku_log_info1_quiet "No Procfile found in app image" - fi -} - -fn-procfile-get-command() { - declare desc="Fetch a command from a Procfile" - declare PROCFILE_PATH="$1" PROCESS_TYPE="$2" PORT="$3" - - if [[ ! -f "$PROCFILE_PATH" ]]; then - return - fi - - procfile-util show --procfile "$PROCFILE_PATH" --process-type "$PROCESS_TYPE" --default-port "$PORT" 2>/dev/null -} - -remove_procfile() { - declare desc="removes DOKKU_PROCFILE" - local APP="$1" - local DOKKU_PROCFILE="$DOKKU_ROOT/$APP/DOKKU_PROCFILE" - if [[ -f "$DOKKU_PROCFILE" ]]; then - rm -f "$DOKKU_PROCFILE" - fi -} - -generate_scale_file() { - declare desc="generates DOKKU_SCALE file" - local APP="$1" - local IMAGE_TAG="$2" - local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" - local DOKKU_PROCFILE="$DOKKU_ROOT/$APP/DOKKU_PROCFILE" - verify_app_name "$APP" - - if copy_from_image "$IMAGE" "DOKKU_SCALE" "$DOKKU_SCALE_FILE" 2>/dev/null; then - cp "$DOKKU_SCALE_FILE" "${DOKKU_SCALE_FILE}.extracted" - else - rm -f "${DOKKU_SCALE_FILE}.extracted" - fi - - if [[ ! -f $DOKKU_SCALE_FILE ]]; then - dokku_log_info1_quiet "DOKKU_SCALE file not found in app image. Generating one based on Procfile..." - - if [[ -f $DOKKU_PROCFILE ]]; then - while read -r line || [[ -n "$line" ]]; do - if [[ -z "$line" ]] || [[ "$line" == "#"* ]]; then - continue - fi - local NAME=${line%%:*} - local NUM_PROCS=0 - [[ "$NAME" == "web" ]] && NUM_PROCS=1 - [[ -n "$NAME" ]] && echo "$NAME=$NUM_PROCS" >>"$DOKKU_SCALE_FILE" - done < <(grep -v -E "^#" "$DOKKU_PROCFILE" | grep ':' | awk -F ':' '{ print $1 }' | sort | uniq) - else - echo "web=1" >>"$DOKKU_SCALE_FILE" - fi - fi - print_dokku_scale_file "$APP" -} - -set_scale() { - declare desc="sets app proc type scaling" - declare APP="$1" - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" - local DOKKU_PROCFILE="$DOKKU_ROOT/$APP/DOKKU_PROCFILE" - shift 1 - - extract_procfile "$APP" "$IMAGE_TAG" >/dev/null - trap "remove_procfile $APP" RETURN INT TERM EXIT - - local SCALE_SETTINGS=("$@") - for procscale in "${SCALE_SETTINGS[@]}"; do - local PROC_NAME=${procscale%%=*} - local PROC_COUNT=${procscale#*=} - if [[ -f "$DOKKU_PROCFILE" ]] && [[ "$PROC_COUNT" != "0" ]] && ! procfile-util exists --procfile "$DOKKU_PROCFILE" --process-type "$PROC_NAME" >/dev/null 2>&1; then - dokku_log_fail "${PROC_NAME} is not a valid process name" - fi - is_number "$PROC_COUNT" || dokku_log_fail "ps:scale $PROC_COUNT is not a number" - dokku_log_info1_quiet "Scaling $APP:$PROC_NAME to $PROC_COUNT" - if (grep -q -E "^${PROC_NAME}=" "$DOKKU_SCALE_FILE" >/dev/null 2>&1); then - sed --in-place "s:^${PROC_NAME}=.*:$PROC_NAME=$PROC_COUNT:g" "$DOKKU_SCALE_FILE" - else - echo "$PROC_NAME=$PROC_COUNT" >>"$DOKKU_SCALE_FILE" - fi - done -} ps_restore() { declare desc="restores app" @@ -139,118 +10,6 @@ ps_restore() { ps_start "$APP" } -ps_start() { - declare desc="starts app" - local APP="$1" - verify_app_name "$APP" - local IMAGE_TAG=$(get_running_image_tag "$APP") - local RUNNING - - if (is_deployed "$APP"); then - plugn trigger pre-start "$APP" - RUNNING="$(fn-ps-is-app-running "$APP")" - - if [[ "$RUNNING" == "mixed" ]]; then - dokku_log_warn "App is running in mixed mode, releasing" - fi - - if [[ "$RUNNING" != "true" ]]; then - plugn trigger release-and-deploy "$APP" "$IMAGE_TAG" - else - dokku_log_warn "App $APP already running" - fi - plugn trigger proxy-build-config "$APP" - else - dokku_log_warn "App $APP has not been deployed" - fi -} - -ps_stop() { - declare desc="stops app" - local APP="$1" - verify_app_name "$APP" - - ! (is_deployed "$APP") && dokku_log_warn "App $APP has not been deployed" && exit 0 - - dokku_log_quiet "Stopping $APP ..." - local DOKKU_SCHEDULER=$(get_app_scheduler "$APP") - plugn trigger scheduler-stop "$DOKKU_SCHEDULER" "$APP" - plugn trigger post-stop "$APP" -} - -ps_rebuild() { - declare desc="rebuilds app from base image" - local APP="$1" - verify_app_name "$APP" - - plugn trigger receive-app "$APP" -} - -ps_restart() { - declare desc="restarts app" - local APP="$1" - verify_app_name "$APP" - local IMAGE_TAG=$(get_running_image_tag "$APP") - - if (is_deployed "$APP"); then - plugn trigger release-and-deploy "$APP" "$IMAGE_TAG" - else - dokku_log_warn "App $APP has not been deployed" - fi -} - -ps_scale() { - declare desc="sets app scaling config according arguments; otherwise according to DOKKU_SCALE file" - local APP="$1" - verify_app_name "$APP" - local IMAGE_TAG=$(get_running_image_tag "$APP") - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" - local DOKKU_PROCFILE="$DOKKU_ROOT/$APP/DOKKU_PROCFILE" - shift 1 - - # shellcheck disable=SC2015 - [[ ! -e $DOKKU_PROCFILE ]] && extract_procfile "$APP" "$IMAGE_TAG" >/dev/null 2>&1 || true - if [[ ! -e $DOKKU_SCALE_FILE ]]; then - generate_scale_file "$APP" "$IMAGE_TAG" >/dev/null 2>&1 - elif [[ -e $DOKKU_PROCFILE ]]; then - for PROC_NAME in $(procfile-util list --procfile "$DOKKU_PROCFILE"); do - if ! grep -qE "^${PROC_NAME}=" "$DOKKU_SCALE_FILE"; then - echo "$PROC_NAME=0" >>"$DOKKU_SCALE_FILE" - fi - done - fi - if [[ -z "$@" ]]; then - dokku_log_info1_quiet "Scaling for $APP" - dokku_col_log_msg_quiet "proctype" "qty" - dokku_col_log_msg_quiet "--------" "---" - while read -r line || [[ -n "$line" ]]; do - [[ -z "$line" ]] && continue - local PROC_NAME=${line%%=*} - local PROC_COUNT=${line#*=} - dokku_col_log_msg "${PROC_NAME}:" "$PROC_COUNT" - done <"$DOKKU_SCALE_FILE" - else - if [[ "$(fn-ps-can-scale "$APP")" == "true" ]]; then - set_scale "$APP" "$@" - plugn trigger release-and-deploy "$APP" "$IMAGE_TAG" - else - dokku_log_fail "App contains DOKKU_SCALE file and cannot be manually scaled" - fi - fi -} - -get_raw_restart_policies() { - declare desc="strips docker options and prints raw restart policies" - local -r phase_file_path=$1 - [[ -r "$phase_file_path" ]] && sed -e '/^--restart=/!d' <"$phase_file_path" -} - -get_restart_policies() { - declare desc="strips docker options and prints restart policies" - local -r phase_file_path=$1 - get_raw_restart_policies "$phase_file_path" | sed -e 's/^--restart=//g' -} - fn-ps-app-status() { declare APP="$1" @@ -263,17 +22,3 @@ fn-ps-is-app-running() { APP_STATUS=$(fn-ps-app-status "$APP") echo "$APP_STATUS" | cut -d ' ' -f 2 } - -fn-ps-can-scale() { - declare APP="$1" - local IMAGE_TAG=$(get_running_image_tag "$APP") - local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" - local CAN_SCALE=false - - [[ ! -e $DOKKU_SCALE_FILE ]] && generate_scale_file "$APP" "$IMAGE_TAG" >/dev/null 2>&1 - if [[ ! -f "${DOKKU_SCALE_FILE}.extracted" ]]; then - CAN_SCALE=true - fi - - echo "$CAN_SCALE" -} diff --git a/plugins/ps/internal-functions b/plugins/ps/internal-functions index 8d863a87e..f21dcb4bc 100755 --- a/plugins/ps/internal-functions +++ b/plugins/ps/internal-functions @@ -6,121 +6,6 @@ source "$PLUGIN_AVAILABLE_PATH/ps/functions" set -eo pipefail [[ $DOKKU_TRACE ]] && set -x -cmd-ps-inspect() { - declare desc="displays a sanitized version of docker inspect for an app" - declare cmd="ps:inspect" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" - - verify_app_name "$APP" - local DOKKU_SCHEDULER=$(get_app_scheduler "$APP") - plugn trigger scheduler-inspect "$DOKKU_SCHEDULER" "$APP" -} - -cmd-ps-report() { - declare desc="displays a ps report for one or more apps" - declare cmd="ps:report" - [[ "$1" == "$cmd" ]] && shift 1 - declare APP="$1" INFO_FLAG="$2" - local INSTALLED_APPS=$(dokku_apps) - - if [[ -n "$APP" ]] && [[ "$APP" == --* ]]; then - INFO_FLAG="$APP" - APP="" - fi - - if [[ -z "$APP" ]] && [[ -z "$INFO_FLAG" ]]; then - INFO_FLAG="true" - fi - - if [[ -z "$APP" ]]; then - for app in $INSTALLED_APPS; do - cmd-ps-report-single "$app" "$INFO_FLAG" | tee || true - done - else - cmd-ps-report-single "$APP" "$INFO_FLAG" - fi -} - -cmd-ps-report-single() { - declare APP="$1" INFO_FLAG="$2" - local APP_DIR="$DOKKU_ROOT/$APP" - [[ "$INFO_FLAG" == "true" ]] && INFO_FLAG="" - - local APP_RUNNING="$(fn-ps-app-status "$APP")" - local PROCS=$(echo "$APP_RUNNING" | cut -d ' ' -f 1) - local RUNNING=$(echo "$APP_RUNNING" | cut -d ' ' -f 2) - - if (is_deployed "$APP"); then DEPLOYED="true"; else DEPLOYED="false"; fi - local passed_phases="deploy" - local RESTARTPOLICY=$(get_restart_policies "$(get_phase_file_path "$passed_phases")" || true) - local CAN_SCALE="$(fn-ps-can-scale "$APP")" - - local CONTAINER_FILES="$(find "$DOKKU_ROOT/$APP" -maxdepth 1 -name "CONTAINER.*" -printf "%f\n" 2>/dev/null | sort -t . -k 2 -n | xargs)" - local CONTAINER_FILE - local DOKKU_APP_RESTORE=$(config_get "$APP" DOKKU_APP_RESTORE || true) - local STATUSES=() - if [[ "$DOKKU_APP_RESTORE" != 0 ]]; then RESTORE="true"; else RESTORE="false"; fi - local APP_CONTAINER_STATUS - for CONTAINER_FILE in $CONTAINER_FILES; do - CID=$(<"$DOKKU_ROOT/$APP/$CONTAINER_FILE") - APP_CONTAINER_STATUS=$("$DOCKER_BIN" container inspect --format '{{.State.Status}}' "$CID" 2>/dev/null || true) - [[ -z "$APP_CONTAINER_STATUS" ]] && APP_CONTAINER_STATUS="missing" - STATUSES+=("${CONTAINER_FILE#*.}:$APP_CONTAINER_STATUS#${CID:0:12}") - done - local flag_map=( - "--processes: $PROCS" - "--deployed: $DEPLOYED" - "--running: $RUNNING" - "--restore: $RESTORE" - "--restart-policy: $RESTARTPOLICY" - "--ps-can-scale: $CAN_SCALE" - ) - - if [[ -z "$INFO_FLAG" ]]; then - dokku_log_info2_quiet "$APP ps information" - for flag in "${flag_map[@]}"; do - key="$(echo "${flag#--}" | cut -f1 -d' ' | tr - ' ')" - dokku_log_verbose "$(printf "%-30s %-25s" "${key^}" "${flag#*: }")" - done - for STATUS in "${STATUSES[@]}"; do - name=${STATUS%:*} - value=${STATUS#*:} - status=${value%#*} - cid=${value#*#} - dokku_log_verbose "$(printf "%-30s %-10s (CID: %s)" "Status $name:" "$status" "$cid")" - done - else - local match=false - local value_exists=false - for STATUS in "${STATUSES[@]}"; do - name=${STATUS%:*} - value=${STATUS#*:} - status=${value%#*} - cid=${value#*#} - status_flag="--status-$name" - flag_map+=("$status_flag: $status") - done - for flag in "${flag_map[@]}"; do - valid_flags="${valid_flags} $(echo "$flag" | cut -d':' -f1)" - if [[ "$flag" == "${INFO_FLAG}:"* ]]; then - value=${flag#*: } - size="${#value}" - if [[ "$size" -ne 0 ]]; then - echo "$value" && match=true && value_exists=true - else - match=true - fi - fi - done - if [[ "$match" == "true" ]]; then - [[ "$value_exists" == "true" ]] || dokku_log_fail "not deployed" - else - dokku_log_fail "Invalid flag passed, valid flags:${valid_flags}" - fi - fi -} - fn-ps-parallel-cmd() { declare desc="Run a ps command against all apps" declare CMD="$1" From 85fafcf170c854db05b4d5d189d742f8a6291c8b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 18:21:35 -0500 Subject: [PATCH 27/67] tests: update tests for new Procfile path --- tests/unit/ps-general.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/ps-general.bats b/tests/unit/ps-general.bats index 534966d42..8f06dc15f 100644 --- a/tests/unit/ps-general.bats +++ b/tests/unit/ps-general.bats @@ -40,16 +40,16 @@ teardown() { @test "(ps:scale) procfile commands extraction" { source "$PLUGIN_CORE_AVAILABLE_PATH/ps/functions" - cat < "$DOKKU_ROOT/$TEST_APP/DOKKU_PROCFILE" + cat < "$DOKKU_LIB_ROOT/data/ps/$TEST_APP/Procfile" web: node web.js --port \$PORT worker: node worker.js EOF - run plugn trigger procfile-get-command "$TEST_APP" web 5001 + PLUGIN_PATH=/var/lib/dokku/plugins PLUGIN_CORE_AVAILABLE_PATH=/var/lib/dokku/core-plugins/available DOKKU_LIB_ROOT=/var/lib/dokku run plugn trigger procfile-get-command "$TEST_APP" web 5001 echo "output: $output" echo "status: $status" assert_output "node web.js --port 5001" - run plugn trigger procfile-get-command "$TEST_APP" worker + PLUGIN_PATH=/var/lib/dokku/plugins PLUGIN_CORE_AVAILABLE_PATH=/var/lib/dokku/core-plugins/available DOKKU_LIB_ROOT=/var/lib/dokku run plugn trigger procfile-get-command "$TEST_APP" worker echo "output: $output" echo "status: $status" assert_output "node worker.js" From 54fb2b5d7513d572bef34e5920c45f4787ddb99f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Nov 2020 18:21:50 -0500 Subject: [PATCH 28/67] feat: suppress output when not necessary --- plugins/ps/triggers.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 63b85e6bd..8f9047941 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -164,7 +164,11 @@ func TriggerProcfileGetCommand(appName string, processType string, port int) err procfilePath := getProcfilePath(appName) if !common.FileExists(procfilePath) { image := common.GetDeployingAppImageName(appName, "", "") - if err := common.PlugnTrigger("procfile-extract", []string{appName, image}...); err != nil { + err := common.SuppressOutput(func() error { + return common.PlugnTrigger("procfile-extract", []string{appName, image}...) + }) + + if err != nil { return err } } From 833a64cc0e308ef4986d1f833e1026ed6cd5a446 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 2 Nov 2020 08:18:24 -0500 Subject: [PATCH 29/67] fix: check if it can scale by checking for an extracted file --- plugins/ps/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index a4ddd52fa..d276d8437 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -14,7 +14,7 @@ import ( ) func canScaleApp(appName string) bool { - return !common.FileExists(getScalefilePath(appName)) + return !common.FileExists(getScalefileExtractedPath(appName)) } func extractProcfile(appName, image string, procfilePath string) error { From bf84405b4321d3ab5e4ec6bfc22827d9dd6c4db6 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 2 Nov 2020 08:18:42 -0500 Subject: [PATCH 30/67] fix: handle scaling properly when there is no procfile --- plugins/ps/functions.go | 18 ++++++++++-------- plugins/ps/subcommands.go | 10 +++++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index d276d8437..9b08842b7 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -223,9 +223,13 @@ func updateScalefile(appName string, processTuples []string) error { return err } - processTypes, err := processesInProcfile(procfilePath) - if err != nil { - return err + processTypes := make(map[string]bool) + hasProcfile := common.FileExists(procfilePath) + if hasProcfile { + processTypes, err = processesInProcfile(procfilePath) + if err != nil { + return err + } } newLines := []string{} @@ -257,10 +261,8 @@ func updateScalefile(appName string, processTuples []string) error { return fmt.Errorf("Invalid count for process type %s", s[0]) } - if _, ok := processTypes[processType]; !ok { - if count != 0 && len(processTypes) == 0 { - return fmt.Errorf("%s is not a valid process name to scale up", processType) - } + if _, ok := processTypes[processType]; !ok && hasProcfile && count != 0 { + return fmt.Errorf("%s is not a valid process name to scale up", processType) } scale = append(scale, fmt.Sprintf("%s=%d", processType, count)) @@ -288,5 +290,5 @@ func updateScalefile(appName string, processTuples []string) error { return err } - return err + return nil } diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 352e3ae78..4438c34c4 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -104,7 +104,10 @@ func CommandScale(appName string, processTuples []string) error { scalefilePath := getScalefilePath(appName) if !common.FileExists(procfilePath) { image := common.GetAppImageRepo(appName) - extractProcfile(appName, image, procfilePath) + common.SuppressOutput(func() error { + extractProcfile(appName, image, procfilePath) + return nil + }) } if !common.FileExists(scalefilePath) { @@ -152,10 +155,15 @@ func CommandScale(appName string, processTuples []string) error { return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) } + common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) if err := updateScalefile(appName, processTuples); err != nil { return err } + if !common.IsDeployed(appName) { + return nil + } + imageTag, err := common.GetRunningImageTag(appName) if err != nil { return err From db435c40e501647d3d95aac15a7acebc8310635f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 7 Nov 2020 13:05:10 -0500 Subject: [PATCH 31/67] fix: do not delete the destination scale file if it was not extracted previously --- plugins/ps/functions.go | 10 +++++++--- plugins/ps/triggers.go | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 9b08842b7..36810de03 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -52,10 +52,14 @@ func extractProcfile(appName, image string, procfilePath string) error { func extractOrGenerateScalefile(appName string, image string) error { destination := getScalefilePath(appName) extracted := getScalefileExtractedPath(appName) - if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", destination); err != nil { + previouslyExtracted := common.FileExists(extracted) + + if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", extracted); err != nil { + if previouslyExtracted { + os.Remove(destination) + } os.Remove(extracted) - os.Remove(destination) - } else if err := common.CopyFile(destination, extracted); err != nil { + } else if err := common.CopyFile(extracted, destination); err != nil { return err } diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 8f9047941..d08558f1d 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -132,6 +132,7 @@ func TriggerPreDeploy(appName string, imageTag string) error { if err := extractProcfile(appName, image, procfilePath); err != nil { return err } + if err := extractOrGenerateScalefile(appName, imageTag); err != nil { return err } From 9193b688545882bb40ba2029924bdd696e092ed2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 7 Nov 2020 22:24:56 -0500 Subject: [PATCH 32/67] hack: enable debug --- tests/ci/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 9a2633f5a..3218cf46e 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -64,7 +64,7 @@ install_dokku() { echo "dokku dokku/skip_key_file boolean true" | sudo debconf-set-selections echo "dokku dokku/vhost_enable boolean true" | sudo debconf-set-selections echo "dokku dokku/web_config boolean false" | sudo debconf-set-selections - sudo dpkg -i "$(cat "${ROOT_DIR}/build/deb-filename")" + sudo TRACE=1 dpkg -i "$(cat "${ROOT_DIR}/build/deb-filename")" } build_dokku_docker_image() { From bb9f764b9fd870a53763ff38d96da7e5e1464e63 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 17 Nov 2020 03:28:03 -0500 Subject: [PATCH 33/67] fix: sync code to bash implementation --- plugins/common/common.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index be763e55a..a211d228c 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -258,17 +258,9 @@ func GetAppImageName(appName, imageTag, imageRepo string) (imageName string) { // IsDeployed returns true if given app has a running container func IsDeployed(appName string) bool { - files, err := ioutil.ReadDir(AppRoot(appName)) - if err != nil { - return false - } - - for _, f := range files { - if f.Name() == "CONTAINER" || strings.HasPrefix(f.Name(), "CONTAINER.") { - return true - } - } - return false + scheduler := GetAppScheduler(appName) + _, err := PlugnTriggerOutput("scheduler-is-deployed", []string{scheduler, appName}...) + return err == nil } // MustGetEnv returns env variable or fails if it's not set From e8d0965a33f9120b469b75514a28ca385d221fe0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 17 Nov 2020 03:28:26 -0500 Subject: [PATCH 34/67] feat: implement all of ps:report --- plugins/ps/functions.go | 7 +++++++ plugins/ps/ps.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 36810de03..67d0f871c 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -152,6 +152,13 @@ func getRestartPolicy(appName string) (string, error) { return "", nil } +func getProcessCount(appName string) (int, error) { + scheduler := common.GetAppScheduler(appName) + b, _ := common.PlugnTriggerOutput("scheduler-app-status", []string{scheduler, appName}...) + count := strings.Split(strings.TrimSpace(string(b[:])), " ")[0] + return strconv.Atoi(count) +} + func getRunningState(appName string) string { scheduler := common.GetAppScheduler(appName) b, _ := common.PlugnTriggerOutput("scheduler-app-status", []string{scheduler, appName}...) diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 201fcad88..9c9b9f012 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -2,6 +2,8 @@ package ps import ( "fmt" + "strconv" + "strings" "github.com/dokku/dokku/plugins/common" ) @@ -72,7 +74,37 @@ func ReportSingleApp(appName, infoFlag string) error { policy = DefaultProperties["restart-policy"] } + canScale := "false" + if canScaleApp(appName) { + canScale = "true" + } + + deployed := "false" + if common.IsDeployed(appName) { + deployed = "true" + } + + runningState := getRunningState(appName) + + count, err := getProcessCount(appName) + if err != nil { + count = -1 + } + + b, _ := common.PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_RESTORE"}...) + restore := strings.TrimSpace(string(b[:])) + if restore == "0" { + restore = "false" + } else { + restore = "true" + } + infoFlags := map[string]string{ + "--processes": strconv.Itoa(count), + "--running": runningState, + "--restore": restore, + "--deployed": deployed, + "--ps-can-scale": canScale, "--ps-restart-policy": policy, } From b4552237b6cfcf827aabea88ec5ccbf61bad6532 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 17 Nov 2020 23:49:06 -0500 Subject: [PATCH 35/67] feat: add ability to specify a plugin build target This speeds up development as it allows us to compile the exact binary needed as opposed to the entire plugin. --- common.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common.mk b/common.mk index a6ad2af00..4f11a8a53 100644 --- a/common.mk +++ b/common.mk @@ -1,4 +1,5 @@ GO_ARGS ?= -a +GO_PLUGIN_MAKE_TARGET ?= build GO_REPO_ROOT := /go/src/github.com/dokku/dokku BUILD_IMAGE := golang:1.14.4 @@ -15,7 +16,7 @@ build-in-docker: clean -e GO111MODULE=on \ -w $(GO_REPO_ROOT)/plugins/$(PLUGIN_NAME) \ $(BUILD_IMAGE) \ - bash -c "GO_ARGS='$(GO_ARGS)' make -j4 build" || exit $$? + bash -c "GO_ARGS='$(GO_ARGS)' make -j4 $(GO_PLUGIN_MAKE_TARGET)" || exit $$? clean: rm -rf commands subcommands triggers prop From cf4a533f2f621f03e18ff67b7e6f462c587d718f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 00:32:16 -0500 Subject: [PATCH 36/67] feat: add common.CatFile() --- plugins/common/io.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/common/io.go b/plugins/common/io.go index 2918284f8..aa22f9d37 100644 --- a/plugins/common/io.go +++ b/plugins/common/io.go @@ -10,6 +10,19 @@ import ( "strings" ) +// CatFile cats the contents of a file (if it exists) +func CatFile(filename string) { + slice, err := FileToSlice(filename) + if err != nil { + LogDebug(fmt.Sprintf("Error cat'ing file %s: %s", filename, err.Error())) + return + } + + for _, line := range slice { + LogDebug(fmt.Sprintf("line: '%s'", line)) + } +} + // CopyFile copies a file from src to dst. If src and dst files exist, and are // the same, then return success. Otherise, attempt to create a hard link // between the two files. If that fail, copy the file contents from src to dst. From f6efb534918d986d0c82fc5699d419d8e2f301c8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 00:32:25 -0500 Subject: [PATCH 37/67] fix: reference image instead of imageTag --- plugins/ps/triggers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index d08558f1d..c192e5517 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -133,7 +133,7 @@ func TriggerPreDeploy(appName string, imageTag string) error { return err } - if err := extractOrGenerateScalefile(appName, imageTag); err != nil { + if err := extractOrGenerateScalefile(appName, image); err != nil { return err } From 0a533c754ff0e926052819a6992d13ac83e3eeff Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 00:33:01 -0500 Subject: [PATCH 38/67] refactor: split ps:scale func into different functions --- plugins/ps/functions.go | 71 ++++++++++++++++++++++++++++++++++++--- plugins/ps/subcommands.go | 56 +++--------------------------- 2 files changed, 70 insertions(+), 57 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 67d0f871c..4b19358eb 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -6,11 +6,13 @@ import ( "fmt" "os" "path/filepath" + "sort" "strconv" "strings" "github.com/dokku/dokku/plugins/common" dockeroptions "github.com/dokku/dokku/plugins/docker-options" + "github.com/ryanuber/columnize" ) func canScaleApp(appName string) bool { @@ -48,7 +50,6 @@ func extractProcfile(appName, image string, procfilePath string) error { return nil } - func extractOrGenerateScalefile(appName string, image string) error { destination := getScalefilePath(appName) extracted := getScalefileExtractedPath(appName) @@ -64,18 +65,20 @@ func extractOrGenerateScalefile(appName string, image string) error { } if common.FileExists(destination) { + common.LogInfo1Quiet("DOKKU_SCALE file exists") return nil } common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") - if err := generateScalefile(appName, destination); err != nil { - common.LogDebug(fmt.Sprintf("Error: %s", err.Error())) + if err := generateScalefile(appName); err != nil { + common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) return err } return nil } -func generateScalefile(appName string, destination string) error { +func generateScalefile(appName string) error { + destination := getScalefilePath(appName) procfilePath := getProcfilePath(appName) content := []string{"web=1"} if !common.FileExists(procfilePath) { @@ -173,6 +176,11 @@ func getScalefileExtractedPath(appName string) string { return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") } +func hasScaleFile(appName string) bool { + scalefilePath := getScalefilePath(appName) + return common.FileExists(scalefilePath) +} + func isValidRestartPolicy(policy string) bool { if policy == "" { return false @@ -226,6 +234,59 @@ func removeProcfile(appName string) error { return os.Remove(procfile) } +func scaleReport(appName string) error { + scalefilePath := getScalefilePath(appName) + lines, err := common.FileToSlice(scalefilePath) + if err != nil { + return err + } + + common.LogInfo1Quiet(fmt.Sprintf("Scaling for %s", appName)) + config := columnize.DefaultConfig() + config.Delim = "=" + config.Glue = ": " + config.Prefix = " " + config.Empty = "" + + content := []string{} + if os.Getenv("DOKKU_QUIET_OUTPUT") == "" { + content = append(content, "proctype=qty", "--------=---") + } + + sort.Strings(lines) + for _, line := range lines { + content = append(content, line) + } + + for _, line := range content { + s := strings.Split(line, "=") + common.Log(fmt.Sprintf("%s %s", common.RightPad(fmt.Sprintf("%s:", s[0]), 5, " "), s[1])) + } + + return nil +} + +func scaleSet(appName string, processTuples []string) error { + if !canScaleApp(appName) { + return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) + } + + common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) + if err := updateScalefile(appName, processTuples); err != nil { + return err + } + + if !common.IsDeployed(appName) { + return nil + } + + imageTag, err := common.GetRunningImageTag(appName) + if err != nil { + return err + } + return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) +} + func updateScalefile(appName string, processTuples []string) error { procfilePath := getProcfilePath(appName) scalefilePath := getScalefilePath(appName) @@ -273,7 +334,7 @@ func updateScalefile(appName string, processTuples []string) error { } if _, ok := processTypes[processType]; !ok && hasProcfile && count != 0 { - return fmt.Errorf("%s is not a valid process name to scale up", processType) + return fmt.Errorf("%s is not a valid process name to scale up", processType) } scale = append(scale, fmt.Sprintf("%s=%d", processType, count)) diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 4438c34c4..131a3b5a7 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -3,14 +3,11 @@ package ps import ( "errors" "fmt" - "os" - "sort" "strings" dockeroptions "github.com/dokku/dokku/plugins/docker-options" "github.com/dokku/dokku/plugins/common" - "github.com/ryanuber/columnize" ) func CommandInspect(appName string) error { @@ -101,7 +98,6 @@ func CommandScale(appName string, processTuples []string) error { } procfilePath := getProcfilePath(appName) - scalefilePath := getScalefilePath(appName) if !common.FileExists(procfilePath) { image := common.GetAppImageRepo(appName) common.SuppressOutput(func() error { @@ -110,9 +106,9 @@ func CommandScale(appName string, processTuples []string) error { }) } - if !common.FileExists(scalefilePath) { + if !hasScaleFile(appName) { err := common.SuppressOutput(func() error { - return generateScalefile(appName, scalefilePath) + return generateScalefile(appName) }) if err != nil { return err @@ -124,54 +120,10 @@ func CommandScale(appName string, processTuples []string) error { } if len(processTuples) == 0 { - lines, err := common.FileToSlice(scalefilePath) - if err != nil { - return err - } - - common.LogInfo1Quiet(fmt.Sprintf("Scaling for %s", appName)) - config := columnize.DefaultConfig() - config.Delim = "=" - config.Glue = ": " - config.Prefix = " " - config.Empty = "" - - content := []string{} - if os.Getenv("DOKKU_QUIET_OUTPUT") == "" { - content = append(content, "proctype=qty","--------=---") - } - - sort.Strings(lines) - for _, line := range lines { - content = append(content, line) - } - - for _, line := range content { - s := strings.Split(line, "=") - common.Log(fmt.Sprintf("%s %s", common.RightPad(fmt.Sprintf("%s:", s[0]), 5, " "), s[1])) - } - } else { - if !canScaleApp(appName) { - return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) - } - - common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) - if err := updateScalefile(appName, processTuples); err != nil { - return err - } - - if !common.IsDeployed(appName) { - return nil - } - - imageTag, err := common.GetRunningImageTag(appName) - if err != nil { - return err - } - return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) + return scaleReport(appName) } - return nil + return scaleSet(appName, processTuples) } func CommandSet(appName string, property string, value string) error { From a8ccc7d2a73fa1a71f3c6d9e3bc8ad2a45c561ff Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 00:36:35 -0500 Subject: [PATCH 39/67] feat: generate the scale file after app creation The web process always starts off at `1`, so this is relatively benign. --- plugins/ps/triggers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index c192e5517..6decd3f80 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -6,10 +6,10 @@ import ( "path/filepath" "strings" + sh "github.com/codeskyblue/go-sh" "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" dockeroptions "github.com/dokku/dokku/plugins/docker-options" - sh "github.com/codeskyblue/go-sh" ) func TriggerAppRestart(appName string) error { @@ -93,7 +93,7 @@ func TriggerPostCreate(appName string) error { return err } - return nil + return generateScalefile(appName) } // TriggerPostDelete destroys the ps properties for a given app container From 23ad4bb2301482214beff7d9d7404c494c074b69 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 01:23:36 -0500 Subject: [PATCH 40/67] refactor: add ability to skip deploys and fix process scaling --- plugins/ps/functions.go | 149 ++++++++++++++-------- plugins/ps/src/subcommands/subcommands.go | 3 +- plugins/ps/subcommands.go | 6 +- 3 files changed, 98 insertions(+), 60 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 4b19358eb..ef12a1b02 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -181,6 +181,11 @@ func hasScaleFile(appName string) bool { return common.FileExists(scalefilePath) } +func hasProcfile(appName string) bool { + procfilePath := getProcfilePath(appName) + return common.FileExists(procfilePath) +} + func isValidRestartPolicy(policy string) bool { if policy == "" { return false @@ -266,13 +271,39 @@ func scaleReport(appName string) error { return nil } -func scaleSet(appName string, processTuples []string) error { +func parseProcessTuples(processTuples []string) (map[string]int, error) { + scale := make(map[string]int) + + for _, processTuple := range processTuples { + s := strings.Split(processTuple, "=") + if len(s) == 1 { + return scale, fmt.Errorf("Missing count for process type %s", processTuple) + } + + processType := s[0] + count, err := strconv.Atoi(s[1]) + if err != nil { + return scale, fmt.Errorf("Invalid count for process type %s", s[0]) + } + + scale[processType] = count + } + + return scale, nil +} + +func scaleSet(appName string, skipDeploy bool, processTuples []string) error { if !canScaleApp(appName) { return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) } common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) - if err := updateScalefile(appName, processTuples); err != nil { + scale, err := parseProcessTuples(processTuples) + if err != nil { + return err + } + + if err := updateScalefile(appName, scale); err != nil { return err } @@ -284,81 +315,87 @@ func scaleSet(appName string, processTuples []string) error { if err != nil { return err } + + if skipDeploy { + return nil + } + return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) } -func updateScalefile(appName string, processTuples []string) error { - procfilePath := getProcfilePath(appName) +func readScaleFile(appName string) (map[string]int, error) { + scale := make(map[string]int) + if !hasScaleFile(appName) { + return scale, errors.New("No DOKKU_SCALE file found") + } + scalefilePath := getScalefilePath(appName) lines, err := common.FileToSlice(scalefilePath) if err != nil { - return err + return scale, err } - processTypes := make(map[string]bool) - hasProcfile := common.FileExists(procfilePath) - if hasProcfile { - processTypes, err = processesInProcfile(procfilePath) - if err != nil { - return err - } - } - - newLines := []string{} - for processType := range processTypes { - found := false - for _, line := range lines { - if strings.HasPrefix(line, fmt.Sprintf("%s=", processType)) { - newLines = append(newLines, line) - found = true - break - } - } - - if !found { - newLines = append(newLines, fmt.Sprintf("%s=0", processType)) - } - } - - scale := []string{} - for _, processTuple := range processTuples { - s := strings.Split(processTuple, "=") - if len(s) == 1 { - return fmt.Errorf("Missing count for process type %s", processTuple) + for i, line := range lines { + s := strings.Split(line, "=") + if len(s) != 2 { + common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i)) + continue } processType := s[0] count, err := strconv.Atoi(s[1]) if err != nil { - return fmt.Errorf("Invalid count for process type %s", s[0]) + common.LogWarn(fmt.Sprintf("Invalid count on line %d, skipping", i)) + continue } - if _, ok := processTypes[processType]; !ok && hasProcfile && count != 0 { + scale[processType] = count + } + + common.LogWarn("No valid entries found in scale file, defaulting to web=1") + if len(scale) == 0 { + scale["web"] = 1 + } + + return scale, nil +} + +func updateScalefile(appName string, scaleUpdates map[string]int) error { + if !hasScaleFile(appName) { + if err := generateScalefile(appName); err != nil { + return err + } + } + + scale, err := readScaleFile(appName) + if err != nil { + return err + } + + procfilePath := getProcfilePath(appName) + procfileExists := hasProcfile(appName) + validProcessTypes := make(map[string]bool) + if procfileExists { + validProcessTypes, err = processesInProcfile(procfilePath) + if err != nil { + return err + } + } + + for processType, count := range scaleUpdates { + if procfileExists && !validProcessTypes[processType] && count != 0 { return fmt.Errorf("%s is not a valid process name to scale up", processType) } - - scale = append(scale, fmt.Sprintf("%s=%d", processType, count)) + scale[processType] = count } - for _, line := range newLines { - s := strings.Split(line, "=") - processType := s[0] - - found := false - for _, s := range scale { - if strings.HasPrefix(s, fmt.Sprintf("%s=", processType)) { - found = true - break - } - } - - if !found { - scale = append(scale, line) - } - + content := []string{} + for processType, count := range scale { + content = append(content, fmt.Sprintf("%s=%d", processType, count)) } - if err := common.WriteSliceToFile(scalefilePath, scale); err != nil { + scalefilePath := getScalefilePath(appName) + if err := common.WriteSliceToFile(scalefilePath, content); err != nil { return err } diff --git a/plugins/ps/src/subcommands/subcommands.go b/plugins/ps/src/subcommands/subcommands.go index cecfc6e75..21be3e9ab 100644 --- a/plugins/ps/src/subcommands/subcommands.go +++ b/plugins/ps/src/subcommands/subcommands.go @@ -55,10 +55,11 @@ func main() { err = ps.CommandRetire() case "scale": args := flag.NewFlagSet("ps:scale", flag.ExitOnError) + skipDeploy := args.Bool("skip-deploy", false, "--skip-deploy: skip deploy of the app") args.Parse(os.Args[2:]) appName := args.Arg(0) _, processTuples := common.ShiftString(args.Args()) - err = ps.CommandScale(appName, processTuples) + err = ps.CommandScale(appName, *skipDeploy, processTuples) case "set": args := flag.NewFlagSet("ps:set", flag.ExitOnError) args.Parse(os.Args[2:]) diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 131a3b5a7..3be60b88d 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -88,7 +88,7 @@ func CommandRetire() error { return nil } -func CommandScale(appName string, processTuples []string) error { +func CommandScale(appName string, skipDeploy bool, processTuples []string) error { if appName == "" { return errors.New("Please specify an app to run the command on") } @@ -114,7 +114,7 @@ func CommandScale(appName string, processTuples []string) error { return err } } else if common.FileExists(procfilePath) { - if err := updateScalefile(appName, []string{}); err != nil { + if err := updateScalefile(appName, make(map[string]int)); err != nil { return err } } @@ -123,7 +123,7 @@ func CommandScale(appName string, processTuples []string) error { return scaleReport(appName) } - return scaleSet(appName, processTuples) + return scaleSet(appName, skipDeploy, processTuples) } func CommandSet(appName string, property string, value string) error { From 82f13af166d147ee2c100595b8de671f1872a903 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 01:24:31 -0500 Subject: [PATCH 41/67] chore: return early --- plugins/ps/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index ef12a1b02..40cdb3deb 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -297,12 +297,12 @@ func scaleSet(appName string, skipDeploy bool, processTuples []string) error { return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) } - common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) scale, err := parseProcessTuples(processTuples) if err != nil { return err } + common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) if err := updateScalefile(appName, scale); err != nil { return err } From 36dc98452d2273ae33827b397f9b4dab7115b3a1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 02:18:08 -0500 Subject: [PATCH 42/67] feat: add function for listing files within a directory by prefix --- plugins/common/io.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/plugins/common/io.go b/plugins/common/io.go index aa22f9d37..93ee48aaa 100644 --- a/plugins/common/io.go +++ b/plugins/common/io.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "io" + "io/ioutil" "os" "os/user" "strconv" @@ -130,6 +131,27 @@ func IsAbsPath(path string) bool { return strings.HasPrefix(path, "/") } +// ListFiles lists files within a given path that have a given prefix +func ListFilesWithPrefix(path string, prefix string) []string { + names, err := ioutil.ReadDir(path) + if err != nil { + return []string{} + } + + files := []string{} + for _, f := range names { + if prefix != "" && !strings.HasPrefix(f.Name(), prefix) { + continue + } + + if f.Mode().IsRegular() { + files = append(files, fmt.Sprintf("%s/%s", path, f.Name())) + } + } + + return files +} + // ReadFirstLine gets the first line of a file that has contents and returns it // if there are no contents, an empty string is returned // will also return an empty string if the file does not exist From f13ee16ab8b32015cc9c4f834dd985b62a01481c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 02:18:25 -0500 Subject: [PATCH 43/67] feat: add support for process state in ps:report output --- plugins/ps/functions.go | 134 +++++++++++++++++++++++----------------- plugins/ps/ps.go | 19 ++++-- 2 files changed, 89 insertions(+), 64 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 40cdb3deb..2041b806a 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -108,6 +108,24 @@ func generateScalefile(appName string) error { return common.WriteSliceToFile(destination, content) } +func getProcessStatus(appName string) map[string]string { + statuses := make(map[string]string) + containerFiles := common.ListFilesWithPrefix(common.AppRoot(appName), "CONTAINER.") + for _, filename := range containerFiles { + containerID := common.ReadFirstLine(filename) + containerStatus, _ := common.DockerInspect(containerID, "{{ .State.Status }}") + process := strings.TrimPrefix(filename, fmt.Sprintf("%s/CONTAINER.", common.AppRoot(appName))) + + if containerStatus == "" { + containerStatus = "missing" + } + + statuses[process] = fmt.Sprintf("%s (CID: %s)", containerStatus, containerID[0:11]) + } + + return statuses +} + func getProcfileCommand(procfilePath string, processType string, port int) (string, error) { if !common.FileExists(procfilePath) { return "", errors.New("No procfile found") @@ -205,6 +223,27 @@ func isValidRestartPolicy(policy string) bool { return strings.HasPrefix(policy, "on-failure:") } +func parseProcessTuples(processTuples []string) (map[string]int, error) { + scale := make(map[string]int) + + for _, processTuple := range processTuples { + s := strings.Split(processTuple, "=") + if len(s) == 1 { + return scale, fmt.Errorf("Missing count for process type %s", processTuple) + } + + processType := s[0] + count, err := strconv.Atoi(s[1]) + if err != nil { + return scale, fmt.Errorf("Invalid count for process type %s", s[0]) + } + + scale[processType] = count + } + + return scale, nil +} + func processesInProcfile(procfilePath string) (map[string]bool, error) { processes := map[string]bool{} @@ -230,6 +269,43 @@ func processesInProcfile(procfilePath string) (map[string]bool, error) { return processes, nil } +func readScaleFile(appName string) (map[string]int, error) { + scale := make(map[string]int) + if !hasScaleFile(appName) { + return scale, errors.New("No DOKKU_SCALE file found") + } + + scalefilePath := getScalefilePath(appName) + lines, err := common.FileToSlice(scalefilePath) + if err != nil { + return scale, err + } + + for i, line := range lines { + s := strings.Split(line, "=") + if len(s) != 2 { + common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i)) + continue + } + + processType := s[0] + count, err := strconv.Atoi(s[1]) + if err != nil { + common.LogWarn(fmt.Sprintf("Invalid count on line %d, skipping", i)) + continue + } + + scale[processType] = count + } + + common.LogWarn("No valid entries found in scale file, defaulting to web=1") + if len(scale) == 0 { + scale["web"] = 1 + } + + return scale, nil +} + func removeProcfile(appName string) error { procfile := getProcfilePath(appName) if !common.FileExists(procfile) { @@ -271,27 +347,6 @@ func scaleReport(appName string) error { return nil } -func parseProcessTuples(processTuples []string) (map[string]int, error) { - scale := make(map[string]int) - - for _, processTuple := range processTuples { - s := strings.Split(processTuple, "=") - if len(s) == 1 { - return scale, fmt.Errorf("Missing count for process type %s", processTuple) - } - - processType := s[0] - count, err := strconv.Atoi(s[1]) - if err != nil { - return scale, fmt.Errorf("Invalid count for process type %s", s[0]) - } - - scale[processType] = count - } - - return scale, nil -} - func scaleSet(appName string, skipDeploy bool, processTuples []string) error { if !canScaleApp(appName) { return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) @@ -323,43 +378,6 @@ func scaleSet(appName string, skipDeploy bool, processTuples []string) error { return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) } -func readScaleFile(appName string) (map[string]int, error) { - scale := make(map[string]int) - if !hasScaleFile(appName) { - return scale, errors.New("No DOKKU_SCALE file found") - } - - scalefilePath := getScalefilePath(appName) - lines, err := common.FileToSlice(scalefilePath) - if err != nil { - return scale, err - } - - for i, line := range lines { - s := strings.Split(line, "=") - if len(s) != 2 { - common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i)) - continue - } - - processType := s[0] - count, err := strconv.Atoi(s[1]) - if err != nil { - common.LogWarn(fmt.Sprintf("Invalid count on line %d, skipping", i)) - continue - } - - scale[processType] = count - } - - common.LogWarn("No valid entries found in scale file, defaulting to web=1") - if len(scale) == 0 { - scale["web"] = 1 - } - - return scale, nil -} - func updateScalefile(appName string, scaleUpdates map[string]int) error { if !hasScaleFile(appName) { if err := generateScalefile(appName); err != nil { diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 9c9b9f012..9aa88e952 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -68,7 +68,6 @@ func ReportSingleApp(appName, infoFlag string) error { return err } - policy, _ := getRestartPolicy(appName) if policy == "" { policy = DefaultProperties["restart-policy"] @@ -100,12 +99,20 @@ func ReportSingleApp(appName, infoFlag string) error { } infoFlags := map[string]string{ - "--processes": strconv.Itoa(count), - "--running": runningState, - "--restore": restore, - "--deployed": deployed, - "--ps-can-scale": canScale, + "--deployed": deployed, + "--processes": strconv.Itoa(count), + "--ps-can-scale": canScale, "--ps-restart-policy": policy, + "--restore": restore, + "--running": runningState, + } + + scheduler := common.GetAppScheduler(appName) + if scheduler == "docker-local" { + processStatus := getProcessStatus(appName) + for process, value := range processStatus { + infoFlags[fmt.Sprintf("--status-%s", process)] = value + } } trimPrefix := false From be26f45b5aaf72d824e1e68633b815e92fd5ea9d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 02:36:49 -0500 Subject: [PATCH 44/67] fix: use correct plugin trigger --- plugins/scheduler-docker-local/scheduler-run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scheduler-docker-local/scheduler-run b/plugins/scheduler-docker-local/scheduler-run index 441165ace..680fc4fab 100755 --- a/plugins/scheduler-docker-local/scheduler-run +++ b/plugins/scheduler-docker-local/scheduler-run @@ -63,7 +63,7 @@ trigger-scheduler-docker-local-scheduler-run() { [[ "$IMAGE_SOURCE_TYPE" == "herokuish" ]] && local EXEC_CMD="/exec" POTENTIAL_PROCFILE_KEY="$1" - PROC_CMD=$(plugn trigger procfile-get-comand "$APP" "$POTENTIAL_PROCFILE_KEY" "5000" 2>/dev/null || echo '') + PROC_CMD=$(plugn trigger procfile-get-command "$APP" "$POTENTIAL_PROCFILE_KEY" "5000" 2>/dev/null || echo '') if [[ -n "$PROC_CMD" ]]; then dokku_log_info1_quiet "Found '$POTENTIAL_PROCFILE_KEY' in Procfile, running that command" From 702e9cf66cbd3ca39bf26201b565269b8a27d9a9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 12:27:56 -0500 Subject: [PATCH 45/67] fix: zero out entries that don't exist in the scale file --- plugins/ps/functions.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 2041b806a..d224e0810 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -407,6 +407,14 @@ func updateScalefile(appName string, scaleUpdates map[string]int) error { scale[processType] = count } + for processType := range validProcessTypes { + count, ok := scale[processType] + if !ok { + count = 0 + } + scale[processType] = count + } + content := []string{} for processType, count := range scale { content = append(content, fmt.Sprintf("%s=%d", processType, count)) From 44ce5c76aedd86e4ec0bc4e22b02c0ca52f5a2bb Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 12:28:12 -0500 Subject: [PATCH 46/67] fix: move log message to correct location --- plugins/ps/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index d224e0810..35cd87939 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -298,8 +298,8 @@ func readScaleFile(appName string) (map[string]int, error) { scale[processType] = count } - common.LogWarn("No valid entries found in scale file, defaulting to web=1") if len(scale) == 0 { + common.LogWarn("No valid entries found in scale file, defaulting to web=1") scale["web"] = 1 } From bc9a60f6964fb6a98e3b658a054789aef2d3cf62 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 12:28:44 -0500 Subject: [PATCH 47/67] feat: switch to updateScalefile from generateScalefile The process is the same. --- plugins/ps/functions.go | 2 +- plugins/ps/subcommands.go | 2 +- plugins/ps/triggers.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 35cd87939..4a6183d6a 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -70,7 +70,7 @@ func extractOrGenerateScalefile(appName string, image string) error { } common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") - if err := generateScalefile(appName); err != nil { + if err := updateScalefile(appName, make(map[string]int)); err != nil { common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) return err } diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 3be60b88d..75182c591 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -108,7 +108,7 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error if !hasScaleFile(appName) { err := common.SuppressOutput(func() error { - return generateScalefile(appName) + return updateScalefile(appName, make(map[string]int)) }) if err != nil { return err diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 6decd3f80..93dd579a9 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -93,7 +93,7 @@ func TriggerPostCreate(appName string) error { return err } - return generateScalefile(appName) + return updateScalefile(appName, make(map[string]int)) } // TriggerPostDelete destroys the ps properties for a given app container From 1b50b4ab0abda7236de07e1c61850c14c822601e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 12:29:03 -0500 Subject: [PATCH 48/67] fix: ensure the scale file has correct processes when it exists --- plugins/ps/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index 4a6183d6a..f95b1f162 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -66,7 +66,7 @@ func extractOrGenerateScalefile(appName string, image string) error { if common.FileExists(destination) { common.LogInfo1Quiet("DOKKU_SCALE file exists") - return nil + return updateScalefile(appName, make(map[string]int)) } common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") From 5d7b2a84f4d58973d61e353d86026c78402851c2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 12:50:59 -0500 Subject: [PATCH 49/67] fix: set DATABASE_URL so dj-database-url properly configures a database config --- tests/unit/app-json.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index 1bd0d2f03..2c4d6517a 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -89,7 +89,7 @@ teardown() { } @test "(app-json) app.json dockerfile release" { - run /bin/bash -c "dokku config:set --no-restart $TEST_APP SECRET_KEY=fjdkslafjdk ENVIRONMENT=dev" + run /bin/bash -c "dokku config:set --no-restart $TEST_APP SECRET_KEY=fjdkslafjdk ENVIRONMENT=dev DATABASE_URL=sqlite:///db.sqlite3" echo "output: $output" echo "status: $status" assert_success From 72ed9dbccee907301bbdb12b09c8b24673191a2f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 12:51:37 -0500 Subject: [PATCH 50/67] fix: only clean things that were built This otherwise break non-golang plugins. --- common.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.mk b/common.mk index 4f11a8a53..f101ffada 100644 --- a/common.mk +++ b/common.mk @@ -19,7 +19,7 @@ build-in-docker: clean bash -c "GO_ARGS='$(GO_ARGS)' make -j4 $(GO_PLUGIN_MAKE_TARGET)" || exit $$? clean: - rm -rf commands subcommands triggers prop + rm -rf $(BUILD) find . -xtype l -delete commands: **/**/commands.go From 037b99865d6bbab901a02a58f0f042934439fd4a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 13:02:02 -0500 Subject: [PATCH 51/67] fix: use CamelCase for constants --- plugins/ps/ps.go | 4 ++-- plugins/ps/src/subcommands/subcommands.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 9aa88e952..830e4dff7 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -8,7 +8,7 @@ import ( "github.com/dokku/dokku/plugins/common" ) -const RUN_IN_SERIAL = -1 +const RunInSerial = -1 type ParallelCommand func(string) error @@ -20,7 +20,7 @@ var ( ) func RunCommandAgainstAllApps(command ParallelCommand, commandName string, runInSerial bool, parallelCount int) error { - if runInSerial && parallelCount != RUN_IN_SERIAL { + if runInSerial && parallelCount != RunInSerial { common.LogWarn("Ignoring --parallel value and running in serial mode") } diff --git a/plugins/ps/src/subcommands/subcommands.go b/plugins/ps/src/subcommands/subcommands.go index 21be3e9ab..a8e976129 100644 --- a/plugins/ps/src/subcommands/subcommands.go +++ b/plugins/ps/src/subcommands/subcommands.go @@ -25,7 +25,7 @@ func main() { case "rebuild": args := flag.NewFlagSet("ps:rebuild", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") - parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) @@ -39,7 +39,7 @@ func main() { case "restart": args := flag.NewFlagSet("ps:restart", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") - parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) @@ -70,7 +70,7 @@ func main() { case "start": args := flag.NewFlagSet("ps:start", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") - parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) @@ -78,7 +78,7 @@ func main() { case "stop": args := flag.NewFlagSet("ps:stop", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") - parallelCount := args.Int("parallel", ps.RUN_IN_SERIAL, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) From b44bc686a18295bdc37875d2aaef6025e477cb2b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 13:20:43 -0500 Subject: [PATCH 52/67] docs: update docs for ps commands --- docs/deployment/process-management.md | 21 +++++++++------------ plugins/ps/src/commands/commands.go | 9 +++++---- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/deployment/process-management.md b/docs/deployment/process-management.md index 3478b608f..82576c672 100644 --- a/docs/deployment/process-management.md +++ b/docs/deployment/process-management.md @@ -3,18 +3,15 @@ > New as of 0.3.14, Enhanced in 0.7.0 ``` -ps:inspect # Displays a sanitized version of docker inspect for an app -ps:rebuild # Rebuild an app from source -ps:rebuildall # Rebuild all apps from source -ps:report [] [] # Displays a process report for one or more apps -ps:restart # Restart app container(s) -ps:restartall # Restart all deployed app containers -ps:scale = [=] # Get/Set how many instances of a given process to run -ps:set # Set or clear a ps property for an app -ps:start # Start app container(s) -ps:startall # Start all deployed app containers -ps:stop # Stop app container(s) -ps:stopall # Stop all app container(s) +ps:inspect # Displays a sanitized version of docker inspect for an app +ps:rebuild [--parallel count] [--serial] [--all|] # Rebuilds an app from source +ps:report [] [] # Displays a process report for one or more apps +ps:restart [--parallel count] [--serial] [--all|] # Restart an app +ps:restore [] # Start previously running apps e.g. after reboot +ps:scale [--skip-deploy] = [=...] # Get/Set how many instances of a given process to run +ps:set # Set or clear a ps property for an app +ps:start [--parallel count] [--serial] [--all|] # Start an app +ps:stop [--parallel count] [--serial] [--all|] # Stop an app ``` By default, Dokku will only start a single `web` process - if defined - though process scaling can be managed by the `ps` plugin or [via a custom `DOKKU_SCALE` file](/docs/deployment/process-management.md#manually-managing-process-scaling). diff --git a/plugins/ps/src/commands/commands.go b/plugins/ps/src/commands/commands.go index 259073d65..7672b4082 100644 --- a/plugins/ps/src/commands/commands.go +++ b/plugins/ps/src/commands/commands.go @@ -19,13 +19,14 @@ Additional commands:` helpContent = ` ps:inspect , Displays a sanitized version of docker inspect for an app - ps:scale = [=...], Get/Set how many instances of a given process to run - ps:start [--parallel count] [--serial] [--all|], Start an app - ps:stop [--parallel count] [--serial] [--all|], Stop an app ps:rebuild [--parallel count] [--serial] [--all|], Rebuilds an app from source ps:report [] [], Displays a process report for one or more apps ps:restart [--parallel count] [--serial] [--all|], Restart an app - ps:restore, Start previously running apps e.g. after reboot + ps:restore [], Start previously running apps e.g. after reboot + ps:scale [--skip-deploy] = [=...], Get/Set how many instances of a given process to run + ps:set , Set or clear a ps property for an app + ps:start [--parallel count] [--serial] [--all|], Start an app + ps:stop [--parallel count] [--serial] [--all|], Stop an app ` ) From f558008574858b63d2964fce928f97ac2c95085e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 13:24:43 -0500 Subject: [PATCH 53/67] fix: correct a few stickler errors --- plugins/common/io.go | 2 +- plugins/docker-options/dockeroptions.go | 1 + plugins/ps/ps.go | 11 ++++++++++- plugins/ps/subcommands.go | 9 +++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/plugins/common/io.go b/plugins/common/io.go index 93ee48aaa..1423bdb32 100644 --- a/plugins/common/io.go +++ b/plugins/common/io.go @@ -131,7 +131,7 @@ func IsAbsPath(path string) bool { return strings.HasPrefix(path, "/") } -// ListFiles lists files within a given path that have a given prefix +// ListFilesWithPrefix lists files within a given path that have a given prefix func ListFilesWithPrefix(path string, prefix string) []string { names, err := ioutil.ReadDir(path) if err != nil { diff --git a/plugins/docker-options/dockeroptions.go b/plugins/docker-options/dockeroptions.go index 1a30faa25..e97386377 100644 --- a/plugins/docker-options/dockeroptions.go +++ b/plugins/docker-options/dockeroptions.go @@ -8,6 +8,7 @@ import ( "strings" ) +// SetDockerOptionForPhases sets an option to specified phases func SetDockerOptionForPhases(appName string, phases []string, name string, value string) error { for _, phase := range phases { if err := touchPhaseFile(appName, phase); err != nil { diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 830e4dff7..d32fa2968 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -8,8 +8,12 @@ import ( "github.com/dokku/dokku/plugins/common" ) +// RunInSerial is the default value for whether to run a command in parallel or not +// and defaults to -1 (false) const RunInSerial = -1 +// ParallelCommand is a type that declares functions +// the ps plugins can execute in parallel type ParallelCommand func(string) error var ( @@ -19,6 +23,7 @@ var ( } ) +// RunCommandAgainstAllApps runs a given ParallelCommand against all apps func RunCommandAgainstAllApps(command ParallelCommand, commandName string, runInSerial bool, parallelCount int) error { if runInSerial && parallelCount != RunInSerial { common.LogWarn("Ignoring --parallel value and running in serial mode") @@ -31,11 +36,15 @@ func RunCommandAgainstAllApps(command ParallelCommand, commandName string, runIn return RunCommandAgainstAllAppsInParallel(command, commandName, parallelCount) } +// RunCommandAgainstAllAppsInParallel runs a given +// ParallelCommand against all apps in parallel // TODO: implement me func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error { return nil } +// RunCommandAgainstAllAppsSerially runs a given +// ParallelCommand against all apps in serial func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error { apps, err := common.DokkuApps() if err != nil { @@ -46,7 +55,7 @@ func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName strin errorCount := 0 for _, appName := range apps { if err = command(appName); err != nil { - errorCount += 1 + errorCount++ } } diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 75182c591..065951a62 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -10,6 +10,7 @@ import ( "github.com/dokku/dokku/plugins/common" ) +// CommandInspect displays a sanitized version of docker inspect for an app func CommandInspect(appName string) error { if appName == "" { return errors.New("Please specify an app to run the command on") @@ -23,6 +24,7 @@ func CommandInspect(appName string) error { return common.PlugnTrigger("scheduler-inspect", []string{scheduler, appName}...) } +// CommandRebuild rebuilds an app from source func CommandRebuild(appName string, allApps bool, runInSerial bool, parallelCount int) error { if allApps { return RunCommandAgainstAllApps(Rebuild, "rebuild", runInSerial, parallelCount) @@ -62,6 +64,7 @@ func CommandReport(appName string, infoFlag string) error { return ReportSingleApp(appName, infoFlag) } +// CommandRestart restarts an app func CommandRestart(appName string, allApps bool, runInSerial bool, parallelCount int) error { if allApps { return RunCommandAgainstAllApps(Restart, "restart", runInSerial, parallelCount) @@ -78,16 +81,19 @@ func CommandRestart(appName string, allApps bool, runInSerial bool, parallelCoun return Restart(appName) } +// CommandRestore starts previously running apps e.g. after reboot // TODO: implement me func CommandRestore(appName string) error { return nil } +// CommandRetire ensures old containers are retired // TODO: implement me func CommandRetire() error { return nil } +// CommandScale gets or sets how many instances of a given process to run func CommandScale(appName string, skipDeploy bool, processTuples []string) error { if appName == "" { return errors.New("Please specify an app to run the command on") @@ -126,6 +132,7 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error return scaleSet(appName, skipDeploy, processTuples) } +// CommandSet sets or clears a ps property for an app func CommandSet(appName string, property string, value string) error { if property == "restart-policy" { if !isValidRestartPolicy(value) { @@ -140,6 +147,7 @@ func CommandSet(appName string, property string, value string) error { return nil } +// CommandStart starts an app func CommandStart(appName string, allApps bool, runInSerial bool, parallelCount int) error { if allApps { return RunCommandAgainstAllApps(Start, "start", runInSerial, parallelCount) @@ -156,6 +164,7 @@ func CommandStart(appName string, allApps bool, runInSerial bool, parallelCount return Start(appName) } +// CommandStop stops an app func CommandStop(appName string, allApps bool, runInSerial bool, parallelCount int) error { if allApps { return RunCommandAgainstAllApps(Stop, "stop", runInSerial, parallelCount) From c9a0f88bf1560ce3b4b28b180c3d22c1593250a1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 18 Nov 2020 13:45:08 -0500 Subject: [PATCH 54/67] fix: correct a few stickler errors --- plugins/ps/triggers.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 93dd579a9..c4affe305 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -12,10 +12,13 @@ import ( dockeroptions "github.com/dokku/dokku/plugins/docker-options" ) +// TriggerAppRestart restarts an app func TriggerAppRestart(appName string) error { return Restart(appName) } +// TriggerCorePostDeploy removes extracted procfiles +// and sets a property to allow the app to be restored on boot func TriggerCorePostDeploy(appName string) error { if err := removeProcfile(appName); err != nil { return err @@ -30,6 +33,7 @@ func TriggerCorePostDeploy(appName string) error { }) } +// TriggerInstall initializes app restart policies func TriggerInstall() error { directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps") if err := os.MkdirAll(directory, 0755); err != nil { @@ -63,6 +67,7 @@ func TriggerInstall() error { return nil } +// TriggerPostAppClone rebuilds the new app func TriggerPostAppClone(oldAppName string, newAppName string) error { if os.Getenv("SKIP_REBUILD") == "true" { return nil @@ -71,6 +76,7 @@ func TriggerPostAppClone(oldAppName string, newAppName string) error { return Rebuild(newAppName) } +// TriggerPostAppRename rebuilds the renamed app func TriggerPostAppRename(oldAppName string, newAppName string) error { if os.Getenv("SKIP_REBUILD") == "true" { return nil @@ -79,6 +85,8 @@ func TriggerPostAppRename(oldAppName string, newAppName string) error { return Rebuild(newAppName) } +// TriggerPostCreate ensures apps have a default restart policy +// and scale value for web func TriggerPostCreate(appName string) error { if err := dockeroptions.AddDockerOptionToPhases(appName, []string{"deploy"}, "--restart=on-failure:10"); err != nil { return err @@ -101,6 +109,7 @@ func TriggerPostDelete(appName string) error { return common.PropertyDestroy("ps", appName) } +// TriggerPostExtract validates a procfile func TriggerPostExtract(appName string, tempWorkDir string) error { procfile := filepath.Join(tempWorkDir, "Procfile") if !common.FileExists(procfile) { @@ -114,6 +123,7 @@ func TriggerPostExtract(appName string, tempWorkDir string) error { return nil } +// TriggerPostStop sets the restore property to false func TriggerPostStop(appName string) error { entries := map[string]string{ "DOKKU_APP_RESTORE": "0", @@ -124,6 +134,7 @@ func TriggerPostStop(appName string) error { }) } +// TriggerPreDeploy ensures an app has an up to date scale file func TriggerPreDeploy(appName string, imageTag string) error { image := common.GetAppImageRepo(appName) removeProcfile(appName) @@ -140,6 +151,7 @@ func TriggerPreDeploy(appName string, imageTag string) error { return nil } +// TriggerProcfileExtract extracted the procfile func TriggerProcfileExtract(appName string, image string) error { directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", appName) if err := os.MkdirAll(directory, 0755); err != nil { @@ -161,6 +173,7 @@ func TriggerProcfileExtract(appName string, image string) error { return extractProcfile(appName, image, procfilePath) } +// TriggerProcfileGetCommand fetches a command from the procfile func TriggerProcfileGetCommand(appName string, processType string, port int) error { procfilePath := getProcfilePath(appName) if !common.FileExists(procfilePath) { @@ -185,6 +198,7 @@ func TriggerProcfileGetCommand(appName string, processType string, port int) err return nil } +// TriggerProcfileRemove removes the procfile if it exists func TriggerProcfileRemove(appName string, procfilePath string) error { if procfilePath == "" { procfilePath = getProcfilePath(appName) From 6e0c376b62f28ef5d3b6d28eec73c464663a83c3 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 02:32:28 -0500 Subject: [PATCH 55/67] fix: correctly handle property and data perms --- plugins/ps/triggers.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index c4affe305..6d2322f71 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -35,6 +35,10 @@ func TriggerCorePostDeploy(appName string) error { // TriggerInstall initializes app restart policies func TriggerInstall() error { + if err := common.PropertySetup("buildpacks"); err != nil { + return fmt.Errorf("Unable to install the buildpacks plugin: %s", err.Error()) + } + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps") if err := os.MkdirAll(directory, 0755); err != nil { return err @@ -106,7 +110,15 @@ func TriggerPostCreate(appName string) error { // TriggerPostDelete destroys the ps properties for a given app container func TriggerPostDelete(appName string) error { - return common.PropertyDestroy("ps", appName) + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", appName) + dataErr := os.RemoveAll(directory) + propertyErr := common.PropertyDestroy("ps", appName) + + if dataErr != nil { + return dataErr + } + + return propertyErr } // TriggerPostExtract validates a procfile From ef083e6289f741d69acea77c28977b7862233755 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 04:37:29 -0500 Subject: [PATCH 56/67] tests: add output and status --- tests/unit/ps-dockerfile-2.bats | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/ps-dockerfile-2.bats b/tests/unit/ps-dockerfile-2.bats index 759f9013e..35dae977f 100644 --- a/tests/unit/ps-dockerfile-2.bats +++ b/tests/unit/ps-dockerfile-2.bats @@ -95,6 +95,8 @@ teardown() { assert_failure run create_app + echo "output: $output" + echo "status: $status" assert_success } From 28a36c490856514f0c2b0f57b891bc859861ec01 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 17:31:13 -0500 Subject: [PATCH 57/67] feat: add proper parallel support --- docs/deployment/process-management.md | 20 ++--- plugins/ps/ps.go | 101 ++++++++++++++++++++-- plugins/ps/src/commands/commands.go | 8 +- plugins/ps/src/subcommands/subcommands.go | 12 +-- plugins/ps/subcommands.go | 16 ++-- 5 files changed, 121 insertions(+), 36 deletions(-) diff --git a/docs/deployment/process-management.md b/docs/deployment/process-management.md index 82576c672..79323ef54 100644 --- a/docs/deployment/process-management.md +++ b/docs/deployment/process-management.md @@ -4,14 +4,14 @@ ``` ps:inspect # Displays a sanitized version of docker inspect for an app -ps:rebuild [--parallel count] [--serial] [--all|] # Rebuilds an app from source +ps:rebuild [--parallel count] [--all|] # Rebuilds an app from source ps:report [] [] # Displays a process report for one or more apps -ps:restart [--parallel count] [--serial] [--all|] # Restart an app +ps:restart [--parallel count] [--all|] # Restart an app ps:restore [] # Start previously running apps e.g. after reboot ps:scale [--skip-deploy] = [=...] # Get/Set how many instances of a given process to run ps:set # Set or clear a ps property for an app -ps:start [--parallel count] [--serial] [--all|] # Start an app -ps:stop [--parallel count] [--serial] [--all|] # Stop an app +ps:start [--parallel count] [--all|] # Start an app +ps:stop [--parallel count] [--all|] # Stop an app ``` By default, Dokku will only start a single `web` process - if defined - though process scaling can be managed by the `ps` plugin or [via a custom `DOKKU_SCALE` file](/docs/deployment/process-management.md#manually-managing-process-scaling). @@ -43,10 +43,10 @@ dokku ps:rebuild node-js-app You may also rebuild all applications at once, which is useful when enabling/disabling a plugin that modifies all applications: ```shell -dokku ps:rebuildall +dokku ps:rebuild --all ``` -> The `ps:rebuild` and `ps:rebuildall` commands only work for applications for which there is a source, and thus +> The `ps:rebuild` command only works for applications for which there is a source, and thus > will only always work deterministically for Git-deployed application. Please see > the [images documentation](/docs/deployment/methods/images.md) and [tar documentation](/docs/deployment/methods/tar.md) > in for more information concerning rebuilding those applications. @@ -62,7 +62,7 @@ dokku ps:restart node-js-app You may also trigger a restart on all applications at one time: ```shell -dokku ps:restartall +dokku ps:restart --all ``` ### `ps:scale` command @@ -104,7 +104,7 @@ dokku ps:stop node-js-app You may also stop all applications at once: ```shell -dokku ps:stopall +dokku ps:stop --all ``` ### Starting applications @@ -117,12 +117,12 @@ dokku ps:start node-js-app ### Starting all applications -In some cases, it may be necessary to start all applications from scratch - e.g. if all Docker containers have been manually stopped. This can be executed via the `ps:startall` command, which supports parallelism in the same manner `ps:rebuildall`, `ps:restartall`, and `ps:stopall` do. +In some cases, it may be necessary to start all applications from scratch - e.g. if all Docker containers have been manually stopped. This can be executed via the `ps:start` command with the `--all` flag, which supports parallelism - via the `--parallel` flag - in the same manner `ps:rebuild`, `ps:restart`, and `ps:stop` do. Be aware that no action will be taken if the application containers are running. ```shell -dokku ps:startall +dokku ps:start --all ``` ## Restart policies diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index d32fa2968..1a9818943 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -2,20 +2,36 @@ package ps import ( "fmt" + "runtime" "strconv" "strings" + "sync" + "time" "github.com/dokku/dokku/plugins/common" ) // RunInSerial is the default value for whether to run a command in parallel or not // and defaults to -1 (false) -const RunInSerial = -1 +const RunInSerial = 0 // ParallelCommand is a type that declares functions // the ps plugins can execute in parallel type ParallelCommand func(string) error +// ParallelCommandRun contains all the arguments +// necessary for a parallel command run +type ParallelCommandRun struct { + AppName string +} + +// ParallelCommandResult is the result of a parallel +// command run +type ParallelCommandResult struct { + AppName string + Error error +} + var ( // DefaultProperties is a map of all valid ps properties with corresponding default property values DefaultProperties = map[string]string{ @@ -24,9 +40,18 @@ var ( ) // RunCommandAgainstAllApps runs a given ParallelCommand against all apps -func RunCommandAgainstAllApps(command ParallelCommand, commandName string, runInSerial bool, parallelCount int) error { - if runInSerial && parallelCount != RunInSerial { - common.LogWarn("Ignoring --parallel value and running in serial mode") +func RunCommandAgainstAllApps(command ParallelCommand, commandName string, parallelCount int) error { + runInSerial := false + + if parallelCount == -1 { + cpuCount := runtime.NumCPU() + common.LogWarn(fmt.Sprintf("Setting --parallel=%d value to CPU count of %d", parallelCount, cpuCount)) + parallelCount = cpuCount + } + + if parallelCount == 0 || parallelCount == 1 { + common.LogWarn(fmt.Sprintf("Running %s in serial mode", commandName)) + runInSerial = true } if runInSerial { @@ -38,9 +63,72 @@ func RunCommandAgainstAllApps(command ParallelCommand, commandName string, runIn // RunCommandAgainstAllAppsInParallel runs a given // ParallelCommand against all apps in parallel -// TODO: implement me func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error { - return nil + apps, err := common.DokkuApps() + if err != nil { + common.LogWarn(err.Error()) + return nil + } + + jobs := make(chan string, parallelCount) + results := make(chan ParallelCommandResult, len(apps)) + + allocate := func() { + for _, appName := range apps { + jobs <- appName + } + close(jobs) + } + + result := func(done chan error) { + var parallelError error + errorCount := 0 + for result := range results { + if result.Error != nil { + common.LogWarn(fmt.Sprintf("Error running %s against app %s", commandName, result.AppName)) + errorCount++ + } + } + if errorCount > 0 { + parallelError = fmt.Errorf("Encountered %d errors during parallel run", errorCount) + } + done <- parallelError + } + + worker := func(wg *sync.WaitGroup, workerID int) { + for appName := range jobs { + common.LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName)) + output := ParallelCommandResult{ + AppName: appName, + Error: command(appName), + } + results <- output + } + wg.Done() + } + + createParallelWorkerPool := func(numberOfWorkers int) { + var wg sync.WaitGroup + for i := 0; i < numberOfWorkers; i++ { + wg.Add(1) + go worker(&wg, i) + } + wg.Wait() + close(results) + } + + startTime := time.Now() + go allocate() + done := make(chan error) + go result(done) + createParallelWorkerPool(parallelCount) + err = <-done + endTime := time.Now() + diff := endTime.Sub(startTime) + + common.LogInfo2Quiet(fmt.Sprintf("Total time taken: %.2f seconds", diff.Seconds())) + + return err } // RunCommandAgainstAllAppsSerially runs a given @@ -54,6 +142,7 @@ func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName strin errorCount := 0 for _, appName := range apps { + common.LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName)) if err = command(appName); err != nil { errorCount++ } diff --git a/plugins/ps/src/commands/commands.go b/plugins/ps/src/commands/commands.go index 7672b4082..d578af0e7 100644 --- a/plugins/ps/src/commands/commands.go +++ b/plugins/ps/src/commands/commands.go @@ -19,14 +19,14 @@ Additional commands:` helpContent = ` ps:inspect , Displays a sanitized version of docker inspect for an app - ps:rebuild [--parallel count] [--serial] [--all|], Rebuilds an app from source + ps:rebuild [--parallel count] [--all|], Rebuilds an app from source ps:report [] [], Displays a process report for one or more apps - ps:restart [--parallel count] [--serial] [--all|], Restart an app + ps:restart [--parallel count] [--all|], Restart an app ps:restore [], Start previously running apps e.g. after reboot ps:scale [--skip-deploy] = [=...], Get/Set how many instances of a given process to run ps:set , Set or clear a ps property for an app - ps:start [--parallel count] [--serial] [--all|], Start an app - ps:stop [--parallel count] [--serial] [--all|], Stop an app + ps:start [--parallel count] [--all|], Start an app + ps:stop [--parallel count] [--all|], Stop an app ` ) diff --git a/plugins/ps/src/subcommands/subcommands.go b/plugins/ps/src/subcommands/subcommands.go index a8e976129..280d08350 100644 --- a/plugins/ps/src/subcommands/subcommands.go +++ b/plugins/ps/src/subcommands/subcommands.go @@ -26,10 +26,9 @@ func main() { args := flag.NewFlagSet("ps:rebuild", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") - runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = ps.CommandRebuild(appName, *allApps, *runInSerial, *parallelCount) + err = ps.CommandRebuild(appName, *allApps, *parallelCount) case "report": args := flag.NewFlagSet("ps:report", flag.ExitOnError) args.Parse(os.Args[2:]) @@ -40,10 +39,9 @@ func main() { args := flag.NewFlagSet("ps:restart", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") - runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = ps.CommandRestart(appName, *allApps, *runInSerial, *parallelCount) + err = ps.CommandRestart(appName, *allApps, *parallelCount) case "restore": args := flag.NewFlagSet("ps:restore", flag.ExitOnError) args.Parse(os.Args[2:]) @@ -71,18 +69,16 @@ func main() { args := flag.NewFlagSet("ps:start", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") - runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = ps.CommandStart(appName, *allApps, *runInSerial, *parallelCount) + err = ps.CommandStart(appName, *allApps, *parallelCount) case "stop": args := flag.NewFlagSet("ps:stop", flag.ExitOnError) allApps := args.Bool("all", false, "--all: restart all apps") parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") - runInSerial := args.Bool("serial", false, "--serial: restart apps serially") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = ps.CommandStop(appName, *allApps, *runInSerial, *parallelCount) + err = ps.CommandStop(appName, *allApps, *parallelCount) default: common.LogFail(fmt.Sprintf("Invalid plugin subcommand call: %s", subcommand)) } diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 065951a62..d4103fb90 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -25,9 +25,9 @@ func CommandInspect(appName string) error { } // CommandRebuild rebuilds an app from source -func CommandRebuild(appName string, allApps bool, runInSerial bool, parallelCount int) error { +func CommandRebuild(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Rebuild, "rebuild", runInSerial, parallelCount) + return RunCommandAgainstAllApps(Rebuild, "rebuild", parallelCount) } if appName == "" { @@ -65,9 +65,9 @@ func CommandReport(appName string, infoFlag string) error { } // CommandRestart restarts an app -func CommandRestart(appName string, allApps bool, runInSerial bool, parallelCount int) error { +func CommandRestart(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Restart, "restart", runInSerial, parallelCount) + return RunCommandAgainstAllApps(Restart, "restart", parallelCount) } if appName == "" { @@ -148,9 +148,9 @@ func CommandSet(appName string, property string, value string) error { } // CommandStart starts an app -func CommandStart(appName string, allApps bool, runInSerial bool, parallelCount int) error { +func CommandStart(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Start, "start", runInSerial, parallelCount) + return RunCommandAgainstAllApps(Start, "start", parallelCount) } if appName == "" { @@ -165,9 +165,9 @@ func CommandStart(appName string, allApps bool, runInSerial bool, parallelCount } // CommandStop stops an app -func CommandStop(appName string, allApps bool, runInSerial bool, parallelCount int) error { +func CommandStop(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Stop, "stop", runInSerial, parallelCount) + return RunCommandAgainstAllApps(Stop, "stop", parallelCount) } if appName == "" { From a19efc1cfa8de5f5b04e48885062eddbd402cfc9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 17:31:25 -0500 Subject: [PATCH 58/67] chore: drop all references to `all` commands where possible. --- contrib/dokku_client.sh | 2 +- docs/development/plugin-triggers.md | 2 +- docs/getting-started/uninstalling.md | 2 +- docs/getting-started/upgrading.md | 8 +++++--- plugins/ps/Makefile | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/contrib/dokku_client.sh b/contrib/dokku_client.sh index b74f954a5..a10c73a0c 100755 --- a/contrib/dokku_client.sh +++ b/contrib/dokku_client.sh @@ -140,7 +140,7 @@ main() { esac [[ " apps certs help ls nginx shell storage trace version " == *" $CMD "* ]] && unset APP - [[ " certs:chain domains:add-global domains:remove-global domains:set-global ps:rebuildall ps:restartall ps:restore ps:startall ps:stopall " == *" $CMD "* ]] && unset APP + [[ " certs:chain domains:add-global domains:remove-global domains:set-global ps:restore " == *" $CMD "* ]] && unset APP [[ "$CMD" =~ events*|plugin*|ssh-keys* ]] && unset APP [[ -n "$APP_ARG" ]] && [[ "$APP_ARG" == "--global" ]] && unset APP [[ -n "$@" ]] && [[ -n "$APP" ]] && app_arg="--app $APP" diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 5a278a6b7..c921bae96 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1254,7 +1254,7 @@ verify_app_name "$APP" ### `post-stop` - Description: Can be used to run commands after an app is manually stopped -- Invoked by: `dokku ps:stop` and `dokku ps:stopall` +- Invoked by: `dokku ps:stop` - Arguments: `$APP` - Example: diff --git a/docs/getting-started/uninstalling.md b/docs/getting-started/uninstalling.md index aa0399a98..480c33f9a 100644 --- a/docs/getting-started/uninstalling.md +++ b/docs/getting-started/uninstalling.md @@ -36,7 +36,7 @@ All applications should be stopped, and all docker containers and images deleted ```shell # stop all applications -dokku ps:stopall +dokku ps:stop --all # cleanup containers and images dokku cleanup diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index 1aceb7440..b4c2ba8b7 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -45,7 +45,9 @@ Why do we recommend stopping all apps? may happen in minor or major releases. ```shell -# for 0.11.4 and newer versions, use: +# for 0.22.0 and newer versions, use + +# for versions between 0.11.4 and 0.21.4, use dokku ps:stopall # for versions between 0.8.1 and 0.11.3, use @@ -61,10 +63,10 @@ After upgrading, you should rebuild the applications to take advantage of any new buildpacks that were released: ```shell -dokku ps:rebuildall +dokku ps:rebuild --all ``` -> If you have any applications deployed via the `tags` or `tar` commands, do not run the `ps:rebuildall` command, +> If you have any applications deployed via the `tags` or `tar` commands, do not run the `ps:rebuild --all` command, > and instead trigger `ps:rebuild` manually for each `git`-deployed application: > > ``` diff --git a/plugins/ps/Makefile b/plugins/ps/Makefile index 004322380..4dccb2f8d 100644 --- a/plugins/ps/Makefile +++ b/plugins/ps/Makefile @@ -1,4 +1,4 @@ -SUBCOMMANDS = subcommands/inspect subcommands/rebuild subcommands/rebuildall subcommands/report subcommands/restart subcommands/restart-policy subcommands/restartall subcommands/restore subcommands/retire subcommands/scale subcommands/set subcommands/start subcommands/startall subcommands/stop subcommands/stopall +SUBCOMMANDS = subcommands/inspect subcommands/rebuild subcommands/report subcommands/restart subcommands/restart-policy subcommands/restore subcommands/retire subcommands/scale subcommands/set subcommands/start subcommands/stop TRIGGERS = triggers/app-restart triggers/install triggers/post-app-clone triggers/post-app-rename triggers/post-create triggers/post-delete triggers/post-stop triggers/pre-deploy triggers/procfile-extract triggers/procfile-get-command triggers/procfile-remove triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = ps From e6e5ede436dd42427003546ddca1a6b89d674f0e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 19:36:13 -0500 Subject: [PATCH 59/67] refactor: make parallel commands available to all of Dokku There are other places in the codebase that currently act on all apps or could benefit from being able to do so. Being able to run `common.RunCommandAgainstAllApps()` and get that for free will be a boon to implementing this functionality elsewhere. --- plugins/common/parallel.go | 135 ++++++++++++++++++++++++++++++++++++ plugins/ps/ps.go | 136 ------------------------------------- plugins/ps/subcommands.go | 8 +-- 3 files changed, 139 insertions(+), 140 deletions(-) create mode 100644 plugins/common/parallel.go diff --git a/plugins/common/parallel.go b/plugins/common/parallel.go new file mode 100644 index 000000000..779bc4f90 --- /dev/null +++ b/plugins/common/parallel.go @@ -0,0 +1,135 @@ +package common + +import ( + "fmt" + "runtime" + "sync" +) + +// ParallelCommand is a type that declares functions +// the ps plugins can execute in parallel +type ParallelCommand func(string) error + +// ParallelCommandRun contains all the arguments +// necessary for a parallel command run +type ParallelCommandRun struct { + Name string +} + +// ParallelCommandResult is the result of a parallel +// command run +type ParallelCommandResult struct { + Name string + Error error +} + +// RunCommandAgainstAllApps runs a given ParallelCommand against all apps +func RunCommandAgainstAllApps(command ParallelCommand, commandName string, parallelCount int) error { + runInSerial := false + + if parallelCount < -1 { + return fmt.Errorf("Invalid value %d for --parallel flag", parallelCount) + } + + if parallelCount == -1 { + cpuCount := runtime.NumCPU() + LogWarn(fmt.Sprintf("Setting --parallel=%d value to CPU count of %d", parallelCount, cpuCount)) + parallelCount = cpuCount + } + + if parallelCount == 0 || parallelCount == 1 { + LogWarn(fmt.Sprintf("Running %s in serial mode", commandName)) + runInSerial = true + } + + if runInSerial { + return runCommandAgainstAllAppsSerially(command, commandName) + } + + return runCommandAgainstAllAppsInParallel(command, commandName, parallelCount) +} + +func runCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error { + apps, err := DokkuApps() + if err != nil { + LogWarn(err.Error()) + return nil + } + + jobs := make(chan string, parallelCount) + results := make(chan ParallelCommandResult, len(apps)) + + go allocateJobs(apps, jobs) + done := make(chan error) + go aggregateResults(results, done) + createParallelWorkerPool(jobs, results, command, parallelCount) + err = <-done + + return err +} + +func runCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error { + apps, err := DokkuApps() + if err != nil { + LogWarn(err.Error()) + return nil + } + + errorCount := 0 + for _, appName := range apps { + LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName)) + if err = command(appName); err != nil { + errorCount++ + } + } + + if errorCount > 0 { + return fmt.Errorf("%s command returned %d errors", commandName, errorCount) + } + + return nil +} + +func allocateJobs(input []string, jobs chan string) { + for _, job := range input { + jobs <- job + } + close(jobs) +} + +func aggregateResults(results chan ParallelCommandResult, done chan error) { + var parallelError error + errorCount := 0 + for result := range results { + if result.Error != nil { + LogWarn(fmt.Sprintf("Error running command against %s", result.Name)) + errorCount++ + } + } + if errorCount > 0 { + parallelError = fmt.Errorf("Encountered %d errors during parallel run", errorCount) + } + done <- parallelError +} + +func createParallelWorker(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, wg *sync.WaitGroup, workerID int) { + for job := range jobs { + LogInfo1(fmt.Sprintf("Running command against %s", job)) + output := ParallelCommandResult{ + Name: job, + Error: command(job), + } + results <- output + } + wg.Done() +} + +func createParallelWorkerPool(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, numberOfWorkers int) { + var wg sync.WaitGroup + for i := 0; i < numberOfWorkers; i++ { + wg.Add(1) + go createParallelWorker(jobs, results, command, &wg, i) + } + wg.Wait() + close(results) +} diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 1a9818943..81c52e8c2 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -2,11 +2,8 @@ package ps import ( "fmt" - "runtime" "strconv" "strings" - "sync" - "time" "github.com/dokku/dokku/plugins/common" ) @@ -15,23 +12,6 @@ import ( // and defaults to -1 (false) const RunInSerial = 0 -// ParallelCommand is a type that declares functions -// the ps plugins can execute in parallel -type ParallelCommand func(string) error - -// ParallelCommandRun contains all the arguments -// necessary for a parallel command run -type ParallelCommandRun struct { - AppName string -} - -// ParallelCommandResult is the result of a parallel -// command run -type ParallelCommandResult struct { - AppName string - Error error -} - var ( // DefaultProperties is a map of all valid ps properties with corresponding default property values DefaultProperties = map[string]string{ @@ -39,122 +19,6 @@ var ( } ) -// RunCommandAgainstAllApps runs a given ParallelCommand against all apps -func RunCommandAgainstAllApps(command ParallelCommand, commandName string, parallelCount int) error { - runInSerial := false - - if parallelCount == -1 { - cpuCount := runtime.NumCPU() - common.LogWarn(fmt.Sprintf("Setting --parallel=%d value to CPU count of %d", parallelCount, cpuCount)) - parallelCount = cpuCount - } - - if parallelCount == 0 || parallelCount == 1 { - common.LogWarn(fmt.Sprintf("Running %s in serial mode", commandName)) - runInSerial = true - } - - if runInSerial { - return RunCommandAgainstAllAppsSerially(command, commandName) - } - - return RunCommandAgainstAllAppsInParallel(command, commandName, parallelCount) -} - -// RunCommandAgainstAllAppsInParallel runs a given -// ParallelCommand against all apps in parallel -func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error { - apps, err := common.DokkuApps() - if err != nil { - common.LogWarn(err.Error()) - return nil - } - - jobs := make(chan string, parallelCount) - results := make(chan ParallelCommandResult, len(apps)) - - allocate := func() { - for _, appName := range apps { - jobs <- appName - } - close(jobs) - } - - result := func(done chan error) { - var parallelError error - errorCount := 0 - for result := range results { - if result.Error != nil { - common.LogWarn(fmt.Sprintf("Error running %s against app %s", commandName, result.AppName)) - errorCount++ - } - } - if errorCount > 0 { - parallelError = fmt.Errorf("Encountered %d errors during parallel run", errorCount) - } - done <- parallelError - } - - worker := func(wg *sync.WaitGroup, workerID int) { - for appName := range jobs { - common.LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName)) - output := ParallelCommandResult{ - AppName: appName, - Error: command(appName), - } - results <- output - } - wg.Done() - } - - createParallelWorkerPool := func(numberOfWorkers int) { - var wg sync.WaitGroup - for i := 0; i < numberOfWorkers; i++ { - wg.Add(1) - go worker(&wg, i) - } - wg.Wait() - close(results) - } - - startTime := time.Now() - go allocate() - done := make(chan error) - go result(done) - createParallelWorkerPool(parallelCount) - err = <-done - endTime := time.Now() - diff := endTime.Sub(startTime) - - common.LogInfo2Quiet(fmt.Sprintf("Total time taken: %.2f seconds", diff.Seconds())) - - return err -} - -// RunCommandAgainstAllAppsSerially runs a given -// ParallelCommand against all apps in serial -func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error { - apps, err := common.DokkuApps() - if err != nil { - common.LogWarn(err.Error()) - return nil - } - - errorCount := 0 - for _, appName := range apps { - common.LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName)) - if err = command(appName); err != nil { - errorCount++ - } - } - - if errorCount > 0 { - return fmt.Errorf("%s command returned %d errors", commandName, errorCount) - } - - return nil -} - // Rebuild rebuilds app from base image func Rebuild(appName string) error { return common.PlugnTrigger("receive-app", []string{appName}...) diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index d4103fb90..bda148ff8 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -27,7 +27,7 @@ func CommandInspect(appName string) error { // CommandRebuild rebuilds an app from source func CommandRebuild(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Rebuild, "rebuild", parallelCount) + return common.RunCommandAgainstAllApps(Rebuild, "rebuild", parallelCount) } if appName == "" { @@ -67,7 +67,7 @@ func CommandReport(appName string, infoFlag string) error { // CommandRestart restarts an app func CommandRestart(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Restart, "restart", parallelCount) + return common.RunCommandAgainstAllApps(Restart, "restart", parallelCount) } if appName == "" { @@ -150,7 +150,7 @@ func CommandSet(appName string, property string, value string) error { // CommandStart starts an app func CommandStart(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Start, "start", parallelCount) + return common.RunCommandAgainstAllApps(Start, "start", parallelCount) } if appName == "" { @@ -167,7 +167,7 @@ func CommandStart(appName string, allApps bool, parallelCount int) error { // CommandStop stops an app func CommandStop(appName string, allApps bool, parallelCount int) error { if allApps { - return RunCommandAgainstAllApps(Stop, "stop", parallelCount) + return common.RunCommandAgainstAllApps(Stop, "stop", parallelCount) } if appName == "" { From 6d514e72faaa935b6819fee1bb0475143d9b3332 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 20:05:29 -0500 Subject: [PATCH 60/67] docs: cleanup headings in nginx plugin --- docs/configuration/nginx.md | 46 ++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/docs/configuration/nginx.md b/docs/configuration/nginx.md index 3aa24e3b9..3bcabb887 100644 --- a/docs/configuration/nginx.md +++ b/docs/configuration/nginx.md @@ -11,7 +11,9 @@ nginx:show-config # Display app nginx config nginx:validate-config [] [--clean] # Validates and optionally cleans up invalid nginx configurations ``` -## Binding to specific addresses +## Usage + +### Binding to specific addresses > New as of 0.19.2 @@ -33,7 +35,7 @@ dokku nginx:set node-js-app bind-address-ipv6 Users with apps that contain a custom `nginx.conf.sigil` file will need to modify the files to respect the new `NGINX_BIND_ADDRESS_IPV4` and `NGINX_BIND_ADDRESS_IPV6` variables. -## HSTS Header +### HSTS Header > New as of 0.20.0 @@ -53,7 +55,7 @@ The following options are also available via the `nginx:set` command: Beware that if you enable the header and a subsequent deploy of your application results in an HTTP deploy (for whatever reason), the way the header works means that a browser will not attempt to request the HTTP version of your site if the HTTPS version fails until the max-age is reached. -## Checking access logs +### Checking access logs You may check nginx access logs via the `nginx:access-logs` command. This assumes that app access logs are being stored in `/var/log/nginx/$APP-access.log`, as is the default in the generated `nginx.conf`. @@ -67,7 +69,7 @@ You may also follow the logs by specifying the `-t` flag. dokku nginx:access-logs node-js-app -t ``` -## Checking error logs +### Checking error logs You may check nginx error logs via the `nginx:access-logs` command. This assumes that app error logs are being stored in `/var/log/nginx/$APP-error.log`, as is the default in the generated `nginx.conf`. @@ -81,7 +83,7 @@ You may also follow the logs by specifying the `-t` flag. dokku nginx:error-logs node-js-app -t ``` -## Changing log path +### Changing log path > New as of 0.20.1 @@ -106,7 +108,7 @@ dokku nginx:set node-js-app error-log-path In all cases, the nginx config must be regenerated after setting the above values. -## Specifying a read timeout +### Specifying a read timeout > New as of 0.21.0 @@ -126,7 +128,7 @@ dokku nginx:set node-js-app proxy-read-timeout In all cases, the nginx config must be regenerated after setting the above value. -## Showing the nginx config +### Showing the nginx config For debugging purposes, it may be useful to show the nginx config. This can be achieved via the `nginx:show-config` command. @@ -134,7 +136,7 @@ For debugging purposes, it may be useful to show the nginx config. This can be a dokku nginx:show-config node-js-app ``` -## Validating nginx configs +### Validating nginx configs It may be desired to validate an nginx config outside of the deployment process. To do so, run the `nginx:validate-config` command. With no arguments, this will validate all app nginx configs, one at a time. A minimal wrapper nginx config is generated for each app's nginx config, upon which `nginx -t` will be run. @@ -158,7 +160,7 @@ The `--clean` flag may also be specified for a given app: dokku nginx:validate-config node-js-app --clean ``` -## Customizing the nginx configuration +### Customizing the nginx configuration > New as of 0.5.0 @@ -187,7 +189,7 @@ dokku nginx:set node-js-app disable-custom-config true Unsetting this value is the same as enabling custom nginx config usage. -### Available template variables +#### Available template variables ``` {{ .APP }} Application name @@ -207,7 +209,7 @@ Finally, each process type has it's network listeners - a list of IP:PORT pairs > Note: Application config variables are available for use in custom templates. To do so, use the form of `{{ var "FOO" }}` to access a variable named `FOO`. -### Customizing via configuration files included by the default templates +#### Customizing via configuration files included by the default templates The default nginx.conf template will include everything from your apps `nginx.conf.d/` subdirectory in the main `server {}` block (see above): @@ -240,7 +242,7 @@ Your `nginx.conf` file - not to be confused with Dokku's `nginx.conf.sigil` - wo Please adjust the `Procfile` and `nginx.conf` file as appropriate. -## Custom Error Pages +### Custom Error Pages By default, Dokku provides custom error pages for the following three categories of errors: @@ -250,7 +252,7 @@ By default, Dokku provides custom error pages for the following three categories These are provided as an alternative to the generic Nginx error page, are shared for _all_ applications, and their contents are located on disk at `/var/lib/dokku/data/nginx-vhosts/dokku-errors`. To customize them for a specific app, create a custom `nginx.conf.sigil` as described above and change the paths to point elsewhere. -## Default site +### Default site By default, Dokku will route any received request with an unknown HOST header value to the lexicographically first site in the nginx config stack. If this is not the desired behavior, you may want to add the following configuration to the global nginx configuration. @@ -294,34 +296,36 @@ The configuration file must be loaded before `/etc/nginx/conf.d/dokku.conf`, so Alternatively, you may push an app to your Dokku host with a name like "00-default". As long as it lists first in `ls /home/dokku/*/nginx.conf | head`, it will be used as the default nginx vhost. -## Domains plugin +## Other + +### Domains plugin See the [domain configuration documentation](/docs/configuration/domains.md) for more information on how to configure domains for your app. -## Customizing hostnames +### Customizing hostnames See the [customizing hostnames documentation](/docs/configuration/domains.md#customizing-hostnames) for more information on how to configure domains for your app. -## Disabling VHOSTS +### Disabling VHOSTS See the [disabling vhosts documentation](/docs/configuration/domains.md#disabling-vhosts) for more information on how to disable domain usage for your app. -## Running behind a load balancer +### Running behind a load balancer See the [load balancer documentation](/docs/configuration/ssl.md#running-behind-a-load-balancer) for more information on how to configure your nginx config for running behind a network load balancer. -## SSL Configuration +### SSL Configuration See the [ssl documentation](/docs/configuration/ssl.md) for more information on how to configure SSL certificates for your application. -## Disabling Nginx +### Disabling Nginx See the [proxy documentation](/docs/advanced-usage/proxy-management.md) for more information on how to disable nginx as the proxy implementation for your app. -## Managing Proxy Port mappings +### Managing Proxy Port mappings See the [proxy documentation](/docs/advanced-usage/proxy-management.md#proxy-port-mapping) for more information on how to manage ports proxied for your app. -## Regenerating nginx config +### Regenerating nginx config See the [proxy documentation](/docs/advanced-usage/proxy-management.md#regenerating-proxy-config) for more information on how to rebuild the nginx proxy configuration for your app. From 246a50a5927da8bf96fed14a7d168144e5767af1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 20:05:47 -0500 Subject: [PATCH 61/67] docs: cleanup process management docs to make usage more clear. --- docs/configuration/nginx.md | 6 + docs/deployment/process-management.md | 176 ++++++++++++++++++-------- 2 files changed, 128 insertions(+), 54 deletions(-) diff --git a/docs/configuration/nginx.md b/docs/configuration/nginx.md index 3bcabb887..36ca66f07 100644 --- a/docs/configuration/nginx.md +++ b/docs/configuration/nginx.md @@ -13,6 +13,12 @@ nginx:validate-config [] [--clean] # Validates and optionally cleans up in ## Usage +### Request Proxying + +By default, the `web` process is the only process proxied by the nginx proxy implementation. Proxying to other process types may be handled by a custom `nginx.conf.sigil` file, as generally described [below](/docs/configuration/nginx.md#customizing-the-nginx-configuration) + +Nginx will proxy the requests in a [round-robin balancing fashion](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream) to the different deployed (scaled) containers running the `web` proctype. This way, the host's resources can be fully leveraged for single-threaded applications (e.g. `dokku ps:scale node-js-app web=4` on a 4-core machine) + ### Binding to specific addresses > New as of 0.19.2 diff --git a/docs/deployment/process-management.md b/docs/deployment/process-management.md index 79323ef54..932346f89 100644 --- a/docs/deployment/process-management.md +++ b/docs/deployment/process-management.md @@ -14,58 +14,99 @@ ps:start [--parallel count] [--all|] # Start an app ps:stop [--parallel count] [--all|] # Stop an app ``` -By default, Dokku will only start a single `web` process - if defined - though process scaling can be managed by the `ps` plugin or [via a custom `DOKKU_SCALE` file](/docs/deployment/process-management.md#manually-managing-process-scaling). - -> The `web` proctype is the only proctype that will invoke custom checks as defined by a `CHECKS` file. It is also the only process type that will be launched in a container that is either proxied via nginx or bound to an external port. In the first case, nginx will proxy the requests in a [round-robin balancing fashion](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream) to the different deployed (scaled) containers running the `web` proctype. This way, the host's resources can be fully leveraged for single-threaded applications (e.g. `dokku ps:scale node-js-app web=4` on a 4-core machine) - ## Usage ### Inspecting app containers > New as of 0.13.0 -A common administrative task to perform is calling `docker inspect` on the containers that are running for an application. This can be an error-prone task to perform, and may also reveal sensitive environment variables if not done correctly. Dokku provides a wrapper around this command via the `ps:inspect` command: +A common administrative task to perform is calling `docker inspect` on the containers that are running for an app. This can be an error-prone task to perform, and may also reveal sensitive environment variables if not done correctly. Dokku provides a wrapper around this command via the `ps:inspect` command: ```shell dokku ps:inspect node-js-app ``` -This command will gather all the running container IDs for your application and call `docker inspect`, sanitizing the output data so it can be copy-pasted elsewhere safely. +This command will gather all the running container IDs for your app and call `docker inspect`, sanitizing the output data so it can be copy-pasted elsewhere safely. -### Rebuilding applications +### Rebuilding apps -There are some Dokku commands which will not automatically rebuild an application's environment, or which can be told to skip a rebuild. For instance, you may wish to run multiple `config:set` commands without a restart so as to speed up configuration. In these cases, you can ultimately trigger an application rebuild using `ps:rebuild` +> Warning: This section applies only to git-based deploys, and only works deterministically for these apps. Please see the [images documentation](/docs/deployment/methods/images.md) and [tar documentation](/docs/deployment/methods/tar.md) +> in for more information concerning rebuilding those apps. + +It may be useful to rebuild an app at will, such as for commands that do not rebuild an app or when skipping a rebuild after setting multiple config values. For these use cases, the `ps:rebuild` function can be used. ```shell dokku ps:rebuild node-js-app ``` -You may also rebuild all applications at once, which is useful when enabling/disabling a plugin that modifies all applications: +All apps may be rebuilt by using the `--all` flag. ```shell dokku ps:rebuild --all ``` -> The `ps:rebuild` command only works for applications for which there is a source, and thus -> will only always work deterministically for Git-deployed application. Please see -> the [images documentation](/docs/deployment/methods/images.md) and [tar documentation](/docs/deployment/methods/tar.md) -> in for more information concerning rebuilding those applications. +By default, rebuilding all apps happens serially. The parallelism may be controlled by the `--parallel` flag. -### Restarting applications +```shell +dokku ps:rebuild --all --parallel 2 +``` -Applications can be restarted, which is functionally identical to calling the `release_and_deploy` function on an application. Please note that any linked containers *must* be started before the application in order to have a successful boot. +Finally, the number of parallel workers may be automatically set to the number of CPUs available by setting the `--parallel` flag to `-1` + +```shell +dokku ps:rebuild --all --parallel -1 +``` + +A missing linked container will result in failure to boot apps. Services should all be started for apps being rebuilt. + +### Restarting apps + +An app may be restarted using the `ps:restart` command. ```shell dokku ps:restart node-js-app ``` -You may also trigger a restart on all applications at one time: +All apps may be restarted by using the `--all` flag. ```shell dokku ps:restart --all ``` -### `ps:scale` command +By default, restarting all apps happens serially. The parallelism may be controlled by the `--parallel` flag. + +```shell +dokku ps:restart --all --parallel 2 +``` + +Finally, the number of parallel workers may be automatically set to the number of CPUs available by setting the `--parallel` flag to `-1` + +```shell +dokku ps:restart --all --parallel -1 +``` + +A missing linked container will result in failure to boot apps. Services should all be started for apps being rebuilt. + +### Displaying existing scale properties + +Issuing the `ps:scale` command with no arguments will output the current scaling properties for an app. + +```shell +dokku ps:scale node-js-app +``` + +``` +-----> Scaling for python +proctype: qty +--------: --- +web: 1 +``` + +### Scaling apps + +#### Via CLI + +> This functionality is disabled if the formation is managed via a file such as `DOKKU_SCALE`. Dokku can also manage scaling itself via the `ps:scale` command. This command can be used to scale multiple process types at the same time. @@ -79,59 +120,101 @@ Multiple process types can be scaled at once: dokku ps:scale node-js-app web=1 worker=1 ``` -Issuing the `ps:scale` command with no process type argument will output the current scaling settings for an application: +If desired, the corresponding deploy will be skipped by using the `--skip-deploy` flag: ```shell -dokku ps:scale node-js-app +dokku ps:scale --skip-deploy node-js-app web=1 ``` -``` ------> Scaling for node-js-app ------> proctype qty ------> -------- --- ------> web 1 ------> worker 1 +#### Manually managing process scaling + +> Using a `DOKKU_SCALE` formation file disables the ability to use `ps:scale` for scaling. + +A `DOKKU_SCALE` file can be committed to the root of the pushed app repository, and must be within the built image artifact in the image's working directory as shown below. + +- Buildpacks: `/app/DOKKU_SCALE` +- Dockerfile: `WORKDIR/DOKKU_SCALE` or `/DOKKU_SCALE` +- Docker Image: `WORKDIR/DOKKU_SCALE` or `/DOKKU_SCALE` + +The `DOKKU_SCALE` file format is as follows: + +```Procfile +web=1 +worker=2 ``` -### Stopping applications +Removing the file will result in Dokku respecting the `ps:scale` command for setting scale values. -Deployed applications can be stopped using the `ps:stop` command. This turns off all running containers for an application, and will result in a **502 Bad Gateway** response for the default nginx proxy implementation. +#### The `web` process + +For initial app deploys, Dokku will default to starting a single `web` process for each app. This process may be defined within the `Procfile` or as the `CMD` (for Dockerfile or Docker image deploys). Scaling of the `web` process - and others - may be managed via `ps:scale` or a `DOKKU_SCALE` formation file either before or after the initial deploy. + +There are also a few other exceptions for the `web` process. + +- Custom checks defined by a `CHECKS` file only apply to the `web` process type. +- By default, the built-in nginx proxy implementation only proxies the `web` process (others may be handled via a custom `nginx.conf.sigil`). + - See the [nginx request proxying documentation](/docs/configuration/nginx.md#request-proxying) for more information on how nginx handles proxied requests. +- Only the `web` process may be bound to an external port. + +### Stopping apps + +Deployed apps can be stopped using the `ps:stop` command. This turns off all running containers for an app, and will result in a **502 Bad Gateway** response for the default nginx proxy implementation. ```shell dokku ps:stop node-js-app ``` -You may also stop all applications at once: +All apps may be stopped by using the `--all` flag. ```shell dokku ps:stop --all ``` -### Starting applications +By default, stopping all apps happens serially. The parallelism may be controlled by the `--parallel` flag. -All stopped containers can be started using the `ps:start` command. This is similar to running `ps:restart`, except no action will be taken if the application containers are running. +```shell +dokku ps:stop --all --parallel 2 +``` + +Finally, the number of parallel workers may be automatically set to the number of CPUs available by setting the `--parallel` flag to `-1` + +```shell +dokku ps:stop --all --parallel -1 +``` + +### Starting apps + +All stopped containers can be started using the `ps:start` command. This is similar to running `ps:restart`, except no action will be taken if the app containers are running. ```shell dokku ps:start node-js-app ``` -### Starting all applications - -In some cases, it may be necessary to start all applications from scratch - e.g. if all Docker containers have been manually stopped. This can be executed via the `ps:start` command with the `--all` flag, which supports parallelism - via the `--parallel` flag - in the same manner `ps:rebuild`, `ps:restart`, and `ps:stop` do. - -Be aware that no action will be taken if the application containers are running. +All apps may be started by using the `--all` flag. ```shell dokku ps:start --all ``` -## Restart policies +By default, starting all apps happens serially. The parallelism may be controlled by the `--parallel` flag. -> New as of 0.7.0 +```shell +dokku ps:start --all --parallel 2 +``` + +Finally, the number of parallel workers may be automatically set to the number of CPUs available by setting the `--parallel` flag to `-1` + +```shell +dokku ps:start --all --parallel -1 +``` + +### Restart policies + +> New as of 0.7.0, Command Changed in 0.22.0 By default, Dokku will automatically restart containers that exit with a non-zero status up to 10 times via the [on-failure Docker restart policy](https://docs.docker.com/engine/reference/run/#restart-policies---restart). -### Setting the restart policy +#### Setting the restart policy > A change in the restart policy must be followed by a `ps:rebuild` call. @@ -154,19 +237,4 @@ dokku ps:set node-js-app restart-policy on-failure dokku ps:set node-js-app restart-policy on-failure:20 ``` -Restart policies have no bearing on server reboot, and Dokku will always attempt to restart your applications at that point unless they were manually stopped. - -## Manually managing process scaling - -You can optionally *commit* a `DOKKU_SCALE` file to the root of your repository - *not* to the `/home/dokku/APP` directory. Dokku expects this file to contain one line for every process defined in your Procfile. - -Example: - -```Procfile -web=1 -worker=2 -``` - -If it is not committed to the repository, the `DOKKU_SCALE` file will otherwise be automatically generated based on your `ps:scale` settings. - -> *NOTE*: Dokku will always use the `DOKKU_SCALE` file that ships with the repo to override any local settings. +Restart policies have no bearing on server reboot, and Dokku will always attempt to restart your apps at that point unless they were manually stopped. From 024ff6e489bdf424f2722fa8590064dffacc48df Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 20:07:05 -0500 Subject: [PATCH 62/67] chore: remove todone :) --- plugins/scheduler-docker-local/scheduler-deploy | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index 24be945b2..0a1a553fc 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -214,7 +214,6 @@ fn-scheduler-docker-local-extract-start-cmd() { return fi - # TODO: remove direct procfile access DOKKU_DOCKERFILE_START_CMD=$(config_get "$APP" DOKKU_DOCKERFILE_START_CMD || true) DOKKU_PROCFILE_START_CMD=$(plugn trigger procfile-get-command "$APP" "$PROC_TYPE" "$PORT") START_CMD=${DOKKU_DOCKERFILE_START_CMD:-$DOKKU_PROCFILE_START_CMD} From 8ae597ad1de9008aff91a81b9e328afc4686b03a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 20:13:26 -0500 Subject: [PATCH 63/67] fix: correct help output --- plugins/ps/src/subcommands/subcommands.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/ps/src/subcommands/subcommands.go b/plugins/ps/src/subcommands/subcommands.go index 280d08350..a1041cbc7 100644 --- a/plugins/ps/src/subcommands/subcommands.go +++ b/plugins/ps/src/subcommands/subcommands.go @@ -24,8 +24,8 @@ func main() { err = ps.CommandInspect(appName) case "rebuild": args := flag.NewFlagSet("ps:rebuild", flag.ExitOnError) - allApps := args.Bool("all", false, "--all: restart all apps") - parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + allApps := args.Bool("all", false, "--all: rebuild all apps") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to rebuild in parallel, -1 to match cpu count") args.Parse(os.Args[2:]) appName := args.Arg(0) err = ps.CommandRebuild(appName, *allApps, *parallelCount) @@ -67,15 +67,15 @@ func main() { err = ps.CommandSet(appName, property, value) case "start": args := flag.NewFlagSet("ps:start", flag.ExitOnError) - allApps := args.Bool("all", false, "--all: restart all apps") - parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + allApps := args.Bool("all", false, "--all: start all apps") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to start in parallel, -1 to match cpu count") args.Parse(os.Args[2:]) appName := args.Arg(0) err = ps.CommandStart(appName, *allApps, *parallelCount) case "stop": args := flag.NewFlagSet("ps:stop", flag.ExitOnError) - allApps := args.Bool("all", false, "--all: restart all apps") - parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restart in parallel, -1 to match cpu count") + allApps := args.Bool("all", false, "--all: stop all apps") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to stop in parallel, -1 to match cpu count") args.Parse(os.Args[2:]) appName := args.Arg(0) err = ps.CommandStop(appName, *allApps, *parallelCount) From ab0cc6c4440fa89204b3a81ac0ce7fe30a81c82b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 20:37:54 -0500 Subject: [PATCH 64/67] fix: implement ps:restore Also add all the niceties around parallelism that were introduced for the other commands. --- plugins/ps/ps.go | 41 +++++++++++++++++++++++ plugins/ps/src/subcommands/subcommands.go | 4 ++- plugins/ps/subcommands.go | 25 ++++++++++++-- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 81c52e8c2..0131fd368 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -92,6 +92,47 @@ func Restart(appName string) error { return common.PlugnTrigger("release-and-deploy", []string{appName}...) } +// Restore ensures an app that should be running is running on boot +func Restore(appName string) error { + scheduler := common.GetAppScheduler(appName) + if err := common.PlugnTrigger("pre-restore", []string{scheduler, appName}...); err != nil { + return fmt.Errorf("Error running pre-restore: %s", err) + } + + if err := common.PlugnTrigger("proxy-clear-config", []string{appName}...); err != nil { + return fmt.Errorf("Error clearing proxy config: %s", err) + } + + if !common.IsDeployed(appName) { + common.LogWarn(fmt.Sprintf("App %s has not been deployed", appName)) + return nil + } + + b, _ := common.PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_RESTORE"}...) + restore := strings.TrimSpace(string(b[:])) + if restore == "0" { + return nil + } + + return Start(appName) +} + +func RestorePrep() error { + apps, err := common.DokkuApps() + if err != nil { + common.LogWarn(err.Error()) + return nil + } + + for _, appName := range apps { + if err := common.PlugnTrigger("proxy-clear-config", []string{appName}...); err != nil { + return fmt.Errorf("Error clearing proxy config: %s", err) + } + } + + return nil +} + // Start starts the app func Start(appName string) error { imageTag, _ := common.GetRunningImageTag(appName) diff --git a/plugins/ps/src/subcommands/subcommands.go b/plugins/ps/src/subcommands/subcommands.go index a1041cbc7..6eec6e0ae 100644 --- a/plugins/ps/src/subcommands/subcommands.go +++ b/plugins/ps/src/subcommands/subcommands.go @@ -44,9 +44,11 @@ func main() { err = ps.CommandRestart(appName, *allApps, *parallelCount) case "restore": args := flag.NewFlagSet("ps:restore", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: restore all apps") + parallelCount := args.Int("parallel", ps.RunInSerial, "--parallel: number of apps to restore in parallel, -1 to match cpu count") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = ps.CommandRestore(appName) + err = ps.CommandRestore(appName, *allApps, *parallelCount) case "retire": args := flag.NewFlagSet("ps:retire", flag.ExitOnError) args.Parse(os.Args[2:]) diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index bda148ff8..0eb22e0c6 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -83,8 +83,29 @@ func CommandRestart(appName string, allApps bool, parallelCount int) error { // CommandRestore starts previously running apps e.g. after reboot // TODO: implement me -func CommandRestore(appName string) error { - return nil +func CommandRestore(appName string, allApps bool, parallelCount int) error { + if allApps { + if err := RestorePrep(); err != nil { + return err + } + + return common.RunCommandAgainstAllApps(Restore, "restore", parallelCount) + } + + if appName == "" { + common.LogWarn("Restore specified without app, assuming --all") + + if err := RestorePrep(); err != nil { + return err + } + return common.RunCommandAgainstAllApps(Restore, "restore", parallelCount) + } + + if err := common.VerifyAppName(appName); err != nil { + return err + } + + return Restore(appName) } // CommandRetire ensures old containers are retired From 5fd1772c3aaf0638fb7f3add4047298e22f1a876 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 20:40:09 -0500 Subject: [PATCH 65/67] chore: drop implemented function --- plugins/ps/subcommands.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 0eb22e0c6..0d4b51398 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -82,7 +82,6 @@ func CommandRestart(appName string, allApps bool, parallelCount int) error { } // CommandRestore starts previously running apps e.g. after reboot -// TODO: implement me func CommandRestore(appName string, allApps bool, parallelCount int) error { if allApps { if err := RestorePrep(); err != nil { From 53a76b135390858b490660aa362e0bdcbc533866 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 20:56:59 -0500 Subject: [PATCH 66/67] feat: implement ps:retire --- plugins/common/common.go | 9 ++++++-- plugins/ps/go.mod | 7 +++++-- plugins/ps/go.sum | 2 ++ plugins/ps/subcommands.go | 21 ++++++++++++++++++- .../scheduler-docker-local/internal-functions | 1 + 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index a211d228c..74f89f85b 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -74,8 +74,13 @@ func GetAppScheduler(appName string) string { return value } - b, _ = PlugnTriggerOutput("config-get-global", []string{"DOKKU_SCHEDULER"}...) - value = string(b[:]) + return GetGlobalScheduler() +} + +// GetGlobalScheduler fetchs the global scheduler +func GetGlobalScheduler() string { + b, _ := PlugnTriggerOutput("config-get-global", []string{"DOKKU_SCHEDULER"}...) + value := string(b[:]) if value != "" { return value } diff --git a/plugins/ps/go.mod b/plugins/ps/go.mod index 8cb14fd8a..466666a27 100644 --- a/plugins/ps/go.mod +++ b/plugins/ps/go.mod @@ -5,11 +5,14 @@ go 1.14 require ( github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 - github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000 - github.com/dokku/dokku/plugins/docker-options v0.0.0-00010101000000-000000000000 + github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000 + github.com/dokku/dokku/plugins/docker-options v0.0.0-00010101000000-000000000000 + github.com/gofrs/flock v0.8.0 github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 ) replace github.com/dokku/dokku/plugins/common => ../common + replace github.com/dokku/dokku/plugins/config => ../config + replace github.com/dokku/dokku/plugins/docker-options => ../docker-options diff --git a/plugins/ps/go.sum b/plugins/ps/go.sum index 8b973f334..b44b50726 100644 --- a/plugins/ps/go.sum +++ b/plugins/ps/go.sum @@ -3,6 +3,8 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcju github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA= github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= github.com/dokku/dokku v0.21.4 h1:WG0hyPzNqKqoQXuI7H/N+cWX9B7mAxTW6Yop40BjCQk= +github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY= +github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/joho/godotenv v1.2.0 h1:vGTvz69FzUFp+X4/bAkb0j5BoLC+9bpqTWY8mjhA9pc= github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 0d4b51398..1847ab9eb 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -3,11 +3,13 @@ package ps import ( "errors" "fmt" + "path/filepath" "strings" dockeroptions "github.com/dokku/dokku/plugins/docker-options" "github.com/dokku/dokku/plugins/common" + "github.com/gofrs/flock" ) // CommandInspect displays a sanitized version of docker inspect for an app @@ -108,8 +110,25 @@ func CommandRestore(appName string, allApps bool, parallelCount int) error { } // CommandRetire ensures old containers are retired -// TODO: implement me func CommandRetire() error { + lockFile := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps", "retire") + scheduler := common.GetGlobalScheduler() + + fileLock := flock.New(lockFile) + locked, err := fileLock.TryLock() + if err != nil { + return fmt.Errorf("Failed to acquire ps:retire lock: %s", err) + } + defer fileLock.Unlock() + + if !locked { + return fmt.Errorf("Failed to acquire ps:retire lock") + } + + if err := common.PlugnTrigger("scheduler-retire", []string{scheduler}...); err != nil { + return err + } + return nil } diff --git a/plugins/scheduler-docker-local/internal-functions b/plugins/scheduler-docker-local/internal-functions index 5635742a3..030714df3 100755 --- a/plugins/scheduler-docker-local/internal-functions +++ b/plugins/scheduler-docker-local/internal-functions @@ -70,6 +70,7 @@ fn-scheduler-docker-local-retire-container() { declare APP="$1" CID="$2" DEAD_TIME="$3" local STATE + dokku_log_info1_quiet "Attempting to retire $APP container $CID" STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)" if [[ -z "$STATE" ]]; then return From ae1b1c7274bb096f133db5ed8ae25a4ece7066a1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 21 Nov 2020 21:07:20 -0500 Subject: [PATCH 67/67] chore: move function to internal --- plugins/ps/functions.go | 16 ++++++++++++++++ plugins/ps/ps.go | 16 ---------------- plugins/ps/subcommands.go | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index f95b1f162..e69dd52e6 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -315,6 +315,22 @@ func removeProcfile(appName string) error { return os.Remove(procfile) } +func restorePrep() error { + apps, err := common.DokkuApps() + if err != nil { + common.LogWarn(err.Error()) + return nil + } + + for _, appName := range apps { + if err := common.PlugnTrigger("proxy-clear-config", []string{appName}...); err != nil { + return fmt.Errorf("Error clearing proxy config: %s", err) + } + } + + return nil +} + func scaleReport(appName string) error { scalefilePath := getScalefilePath(appName) lines, err := common.FileToSlice(scalefilePath) diff --git a/plugins/ps/ps.go b/plugins/ps/ps.go index 0131fd368..b6849c189 100644 --- a/plugins/ps/ps.go +++ b/plugins/ps/ps.go @@ -117,22 +117,6 @@ func Restore(appName string) error { return Start(appName) } -func RestorePrep() error { - apps, err := common.DokkuApps() - if err != nil { - common.LogWarn(err.Error()) - return nil - } - - for _, appName := range apps { - if err := common.PlugnTrigger("proxy-clear-config", []string{appName}...); err != nil { - return fmt.Errorf("Error clearing proxy config: %s", err) - } - } - - return nil -} - // Start starts the app func Start(appName string) error { imageTag, _ := common.GetRunningImageTag(appName) diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 1847ab9eb..ab085160d 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -86,7 +86,7 @@ func CommandRestart(appName string, allApps bool, parallelCount int) error { // CommandRestore starts previously running apps e.g. after reboot func CommandRestore(appName string, allApps bool, parallelCount int) error { if allApps { - if err := RestorePrep(); err != nil { + if err := restorePrep(); err != nil { return err } @@ -96,7 +96,7 @@ func CommandRestore(appName string, allApps bool, parallelCount int) error { if appName == "" { common.LogWarn("Restore specified without app, assuming --all") - if err := RestorePrep(); err != nil { + if err := restorePrep(); err != nil { return err } return common.RunCommandAgainstAllApps(Restore, "restore", parallelCount)