From 5d089646992aab4b0bffd3804fde9fcbfd276747 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 17 Jul 2020 19:45:05 -0400 Subject: [PATCH 01/43] feat: add AppHostRoot() function This makes it a bit easier to fetch files from the host path --- plugins/common/common.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/common/common.go b/plugins/common/common.go index b34cee5bc..9dfa5c811 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -80,6 +80,12 @@ func AppRoot(appName string) string { return fmt.Sprintf("%v/%v", dokkuRoot, appName) } +// AppRoot returns the app root path +func AppHostRoot(appName string) string { + dokkuHostRoot := MustGetEnv("DOKKU_HOST_ROOT") + return fmt.Sprintf("%v/%v", dokkuHostRoot, appName) +} + // AskForDestructiveConfirmation checks for confirmation on destructive actions func AskForDestructiveConfirmation(name string, objectType string) error { LogWarn("WARNING: Potentially Destructive Action") From 0351bf73b09b4e7d96cc19f14c1747cc721f09ce Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 17 Jul 2020 19:45:52 -0400 Subject: [PATCH 02/43] fix: update deprecated function name --- plugins/common/functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/common/functions b/plugins/common/functions index d377888b1..85fb0cbb6 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -541,7 +541,7 @@ is_app_running() { declare deprecated=true local APP="$1" - dokku_log_warn "Deprecated: ps#fn-is-app-running" + dokku_log_warn "Deprecated: ps#fn-ps-is-app-running" verify_app_name "$APP" local APP_RUNNING_CONTAINER_IDS=$(get_app_running_container_ids "$APP" 2>/dev/null) From dee879538c8210912ba58d20fce6aa6283ca69de Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 17 Jul 2020 19:46:02 -0400 Subject: [PATCH 03/43] feat; add common.LogExclaim() --- plugins/common/log.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/common/log.go b/plugins/common/log.go index 7aa77b82b..065eceb72 100644 --- a/plugins/common/log.go +++ b/plugins/common/log.go @@ -76,6 +76,11 @@ func LogWarn(text string) { fmt.Fprintln(os.Stderr, fmt.Sprintf(" ! %s", text)) } +// LogExclaim is the log exclaim formatter +func LogExclaim(text string) { + fmt.Println(fmt.Sprintf(" ! %s", text)) +} + // LogStderr is the stderr log formatter func LogStderr(text string) { fmt.Fprintln(os.Stderr, text) From 82a6dbf887d29677842ae921ec759622dc837626 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 18 Jul 2020 13:57:06 -0400 Subject: [PATCH 04/43] fix: correct variable declaration --- plugins/app-json/internal-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-json/internal-functions b/plugins/app-json/internal-functions index 3f866a266..e7c12e605 100755 --- a/plugins/app-json/internal-functions +++ b/plugins/app-json/internal-functions @@ -7,7 +7,7 @@ source "$PLUGIN_AVAILABLE_PATH/ps/functions" get_phase_script() { declare desc="extracts app.json from app image and returns the appropriate json key/value" - declare IMAGE_TAG="$1" PHASE_SCRIPT_KEY="$2" + declare IMAGE="$1" PHASE_SCRIPT_KEY="$2" local APP_JSON_FILE=$(mktemp "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") trap "rm -rf '$APP_JSON_FILE' >/dev/null" RETURN INT TERM From 6f6fa709ec8bee319f6b8c058c39a09bd6ab463d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 18 Jul 2020 13:58:05 -0400 Subject: [PATCH 05/43] feat: implement common.CopyFromImage --- plugins/common/common.go | 129 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/plugins/common/common.go b/plugins/common/common.go index 9dfa5c811..c4d072c0f 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -118,6 +118,107 @@ func CommandUsage(helpHeader string, helpContent string) { fmt.Println(columnize.Format(content, config)) } +// CopyFromImage copies a file from named image to destination +func CopyFromImage(appName string, image string, source string, destination string) error { + if err := VerifyAppName(appName); err != nil { + return err + } + + if !VerifyImage(image) { + return fmt.Errorf("Invalid docker image for copying content") + } + + workDir := "" + if !IsAbsPath(source) { + if IsImageHerokuishBased(image, appName) { + workDir = "/app" + } else { + workDir, _ = DockerInspect(image, "{{.Config.WorkingDir}}") + } + + if workDir != "" { + source = fmt.Sprintf("%s/%s", workDir, source) + } + } + + tmpFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", MustGetEnv("DOKKU_PID"), "CopyFromImage")) + if err != nil { + return fmt.Errorf("Cannot create temporary file: %v", err) + } + + defer tmpFile.Close() + defer os.Remove(tmpFile.Name()) + + globalRunArgs := MustGetEnv("DOKKU_GLOBAL_RUN_ARGS") + createLabelArgs := []string{"--label", fmt.Sprintf("com.dokku.app-name=%s", appName), globalRunArgs} + containerID, err := DockerContainerCreate(image, createLabelArgs) + if err != nil { + return fmt.Errorf("Unable to create temporary container: %v", err) + } + + // docker cp exits with status 1 when run as non-root user when it tries to chown the file + // after successfully copying the file. Thus, we suppress stderr. + // ref: https://github.com/dotcloud/docker/issues/3986 + containerCopyCmd := NewShellCmd(strings.Join([]string{ + DockerBin(), + "container", + "cp", + fmt.Sprintf("%s:%s", containerID, source), + tmpFile.Name(), + }, " ")) + containerCopyCmd.ShowOutput = false + fileCopied := containerCopyCmd.Execute() + + containerRemoveCmd := NewShellCmd(strings.Join([]string{ + DockerBin(), + "container", + "rm", + "--force", + containerID, + }, " ")) + containerRemoveCmd.ShowOutput = false + containerRemoveCmd.Execute() + + if !fileCopied { + return fmt.Errorf("Unable to copy file %s from image", source) + } + + fi, err := os.Stat(tmpFile.Name()) + if err != nil { + return err + } + + if fi.Size() == 0 { + return fmt.Errorf("Unable to copy file %s from image", source) + } + + // workaround for CHECKS file when owner is root. seems to only happen when running inside docker + dos2unixCmd := NewShellCmd(strings.Join([]string{ + "dos2unix", + "-l", + "-n", + tmpFile.Name(), + destination, + }, " ")) + dos2unixCmd.ShowOutput = false + dos2unixCmd.Execute() + + // add trailing newline for certain places where file parsing depends on it + b, err := sh.Command("tail", "-c1", destination).Output() + if string(b[:]) != "" { + f, err := os.OpenFile(destination, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + if _, err := f.WriteString("\n"); err != nil { + return fmt.Errorf("Unable to append trailing newline to copied file: %v", err) + } + } + + return nil +} + // DockerCleanup cleans up all exited/dead containers and removes all dangling images func DockerCleanup(appName string, forceCleanup bool) error { if !forceCleanup { @@ -337,6 +438,29 @@ func DirectoryExists(filePath string) bool { return fi.IsDir() } +// DockerContainerCreate creates a new container and returns the container ID +func DockerContainerCreate(image string, containerCreateArgs []string) (string, error) { + cmd := []string{ + DockerBin(), + "container", + "create", + } + + cmd = append(cmd, containerCreateArgs...) + cmd = append(cmd, image) + + var stderr bytes.Buffer + containerCreateCmd := NewShellCmd(strings.Join(cmd, " ")) + containerCreateCmd.ShowOutput = false + containerCreateCmd.Command.Stderr = &stderr + b, err := containerCreateCmd.Output() + if err != nil { + return "", errors.New(strings.TrimSpace(stderr.String())) + } + + return string(b[:]), nil +} + // DockerInspect runs an inspect command with a given format against a container id func DockerInspect(containerOrImageID, format string) (output string, err error) { b, err := sh.Command(DockerBin(), "inspect", "--format", format, containerOrImageID).Output() @@ -430,6 +554,11 @@ func GetAppImageName(appName, imageTag, imageRepo string) (imageName string) { return } +// IsAbsPath returns 0 if input path is absolute +func IsAbsPath(path string) bool { + return strings.HasPrefix(path, "/") +} + // IsDeployed returns true if given app has a running container func IsDeployed(appName string) bool { files, err := ioutil.ReadDir(AppRoot(appName)) From b6d5d291b054bf1f5dd9a9d937c2ccec25343159 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 18 Jul 2020 13:59:37 -0400 Subject: [PATCH 06/43] feat: template out app.json plugin Note: this won't compile as the triggers themselves aren't implemented just yet. --- plugins/app-json/.gitignore | 4 ++++ plugins/app-json/Makefile | 5 +++++ plugins/app-json/go.mod | 11 +++++++++++ plugins/app-json/go.sum | 6 ++++++ 4 files changed, 26 insertions(+) create mode 100644 plugins/app-json/.gitignore create mode 100644 plugins/app-json/Makefile create mode 100644 plugins/app-json/go.mod create mode 100644 plugins/app-json/go.sum diff --git a/plugins/app-json/.gitignore b/plugins/app-json/.gitignore new file mode 100644 index 000000000..d04745dda --- /dev/null +++ b/plugins/app-json/.gitignore @@ -0,0 +1,4 @@ +/triggers/* +/triggers +/pre-* +/post-* diff --git a/plugins/app-json/Makefile b/plugins/app-json/Makefile new file mode 100644 index 000000000..1640a6195 --- /dev/null +++ b/plugins/app-json/Makefile @@ -0,0 +1,5 @@ +TRIGGERS = triggers/post-deploy triggers/pre-deploy +BUILD = triggers +PLUGIN_NAME = app-json + +include ../../common.mk diff --git a/plugins/app-json/go.mod b/plugins/app-json/go.mod new file mode 100644 index 000000000..41b96b55e --- /dev/null +++ b/plugins/app-json/go.mod @@ -0,0 +1,11 @@ +module github.com/dokku/dokku/plugins/apps-json + +go 1.14 + +require ( + github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect + github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 + github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect +) + +replace github.com/dokku/dokku/plugins/common => ../common diff --git a/plugins/app-json/go.sum b/plugins/app-json/go.sum new file mode 100644 index 000000000..f05068f8d --- /dev/null +++ b/plugins/app-json/go.sum @@ -0,0 +1,6 @@ +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= From 98bc513336706eddf531a27a6e341b2d9d3e430a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 18 Jul 2020 14:02:53 -0400 Subject: [PATCH 07/43] feat: implement getPhaseScript and scaffold out plugin triggers --- plugins/app-json/appjson.go | 47 +++++++++++++++++++++++ plugins/app-json/post-deploy | 16 -------- plugins/app-json/pre-deploy | 17 -------- plugins/app-json/src/triggers/triggers.go | 36 +++++++++++++++++ plugins/app-json/triggers.go | 19 +++++++++ 5 files changed, 102 insertions(+), 33 deletions(-) create mode 100644 plugins/app-json/appjson.go delete mode 100755 plugins/app-json/post-deploy delete mode 100755 plugins/app-json/pre-deploy create mode 100644 plugins/app-json/src/triggers/triggers.go create mode 100644 plugins/app-json/triggers.go diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go new file mode 100644 index 000000000..b632ef67e --- /dev/null +++ b/plugins/app-json/appjson.go @@ -0,0 +1,47 @@ +package appjson + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + + "github.com/dokku/dokku/plugins/common" +) + +type AppJson struct { + Scripts struct { + Dokku string `json:"dokku"` + } `json:"scripts"` +} + +// getPhaseScript extracts app.json from app image and returns the appropriate json key/value +func getPhaseScript(appName string, image string, phase string) (string, error) { + appJsonFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "getPhaseScript")) + if err != nil { + return "", fmt.Errorf("Cannot create temporary file: %v", err) + } + + defer os.Remove(appJsonFile.Name()) + + common.CopyFromImage(appName, image, "app.json", appJsonFile.Name()) + if !common.FileExists(appJsonFile.Name()) { + return "", nil + } + + b, err := ioutil.ReadAll(appJsonFile) + if err != nil { + return "", fmt.Errorf("Cannot read app.json file: %v", err) + } + + var appJson AppJson + if err = json.Unmarshal(b, &appJson); err != nil { + return "", fmt.Errorf("Cannot parse app.json: %v", err) + } + + return appJson.Scripts.Dokku, nil +} + +func executeScript(appName string, imageTag string, phase string) error { + return nil +} diff --git a/plugins/app-json/post-deploy b/plugins/app-json/post-deploy deleted file mode 100755 index 3d2eef033..000000000 --- a/plugins/app-json/post-deploy +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_CORE_AVAILABLE_PATH/app-json/internal-functions" - -trigger-app-json-post-deploy() { - declare desc="app-json scripts execution" - declare trigger="post-deploy" - declare APP="$1" IMAGE_TAG="$4" - local PHASE_SCRIPT_KEY="postdeploy" - - execute_script "$APP" "$IMAGE_TAG" "$PHASE_SCRIPT_KEY" -} - -trigger-app-json-post-deploy "$@" diff --git a/plugins/app-json/pre-deploy b/plugins/app-json/pre-deploy deleted file mode 100755 index b9ac30bf0..000000000 --- a/plugins/app-json/pre-deploy +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_CORE_AVAILABLE_PATH/app-json/internal-functions" - -trigger-app-json-pre-deploy() { - declare desc="app-json scripts execution" - declare trigger="pre-deploy" - declare APP="$1" IMAGE_TAG="$2" - local PHASE_SCRIPT_KEY="predeploy" - - execute_script "$APP" "$IMAGE_TAG" "$PHASE_SCRIPT_KEY" - execute_script "$APP" "$IMAGE_TAG" "release" -} - -trigger-app-json-pre-deploy "$@" diff --git a/plugins/app-json/src/triggers/triggers.go b/plugins/app-json/src/triggers/triggers.go new file mode 100644 index 000000000..200ae162e --- /dev/null +++ b/plugins/app-json/src/triggers/triggers.go @@ -0,0 +1,36 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/app-json" + "github.com/dokku/dokku/plugins/common" +) + +// main entrypoint to all triggers +func main() { + parts := strings.Split(os.Args[0], "/") + trigger := parts[len(parts)-1] + flag.Parse() + + var err error + switch trigger { + case "post-deploy": + appName := flag.Arg(0) + imageTag := flag.Arg(3) + err = appjson.TriggerPostDeploy(appName, imageTag) + case "pre-deploy": + appName := flag.Arg(0) + imageTag := flag.Arg(1) + err = appjson.TriggerPreDeploy(appName, imageTag) + default: + common.LogFail(fmt.Sprintf("Invalid plugin trigger call: %s", trigger)) + } + + if err != nil { + common.LogFail(err.Error()) + } +} diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go new file mode 100644 index 000000000..d802a92c2 --- /dev/null +++ b/plugins/app-json/triggers.go @@ -0,0 +1,19 @@ +package appjson + +func TriggerPostDeploy(appName string, imageTag string) error { + if err := executeScript(appName, imageTag, "predeploy"); err != nil { + return err + } + + if err := executeScript(appName, imageTag, "release"); err != nil { + return err + } + return nil +} + +func TriggerPreDeploy(appName string, imageTag string) error { + if err := executeScript(appName, imageTag, "postdeploy"); err != nil { + return err + } + return nil +} From 1ea60a16e57e1258336d6144e632bcf52b83971c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 1 Aug 2020 21:56:19 -0400 Subject: [PATCH 08/43] refactor: move external procfile manipulation into triggers This also cleans up the internal APIs a bit. --- docs/development/plugin-triggers.md | 45 +++++++++++++++++++++++++++ plugins/20_events/procfile-extract | 1 + plugins/20_events/procfile-remove | 1 + plugins/20_events/profile-get-command | 1 + plugins/ps/functions | 27 +++++++++++++--- plugins/ps/procfile-extract | 23 ++++++++++++++ plugins/ps/procfile-get-command | 19 +++++++++++ plugins/ps/procfile-remove | 18 +++++++++++ 8 files changed, 130 insertions(+), 5 deletions(-) create mode 120000 plugins/20_events/procfile-extract create mode 120000 plugins/20_events/procfile-remove create mode 120000 plugins/20_events/profile-get-command create mode 100755 plugins/ps/procfile-extract create mode 100755 plugins/ps/procfile-get-command create mode 100755 plugins/ps/procfile-remove diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 05dd41e2b..660b0696e 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1447,6 +1447,51 @@ APP="$1"; curl "https://dokku.me/starting/${APP}" || true ``` +### `procfile-extract` + +- Description: Extracts a Procfile from a given image to a path +- Invoked by: `internally` +- Arguments: `$APP $IMAGE $FORCE_EXTRACT $PROCFILE_PATH` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + +### `procfile-get-command` + +- Description: Fetches the command for a specific process type +- Invoked by: `internally` +- Arguments: `$APP $PROCESS_TYPE $PORT $PROCFILE_PATH` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + +### `procfile-remove` + +- Description: Removes the extracted Procfile +- Invoked by: `internally` +- Arguments: `$APP $PROCFILE_PATH` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +# TODO +``` + ### `proxy-build-config` - Description: Builds the proxy implementation configuration for a given app diff --git a/plugins/20_events/procfile-extract b/plugins/20_events/procfile-extract new file mode 120000 index 000000000..5178a749f --- /dev/null +++ b/plugins/20_events/procfile-extract @@ -0,0 +1 @@ +hook \ No newline at end of file diff --git a/plugins/20_events/procfile-remove b/plugins/20_events/procfile-remove new file mode 120000 index 000000000..5178a749f --- /dev/null +++ b/plugins/20_events/procfile-remove @@ -0,0 +1 @@ +hook \ No newline at end of file diff --git a/plugins/20_events/profile-get-command b/plugins/20_events/profile-get-command new file mode 120000 index 000000000..5178a749f --- /dev/null +++ b/plugins/20_events/profile-get-command @@ -0,0 +1 @@ +hook \ No newline at end of file diff --git a/plugins/ps/functions b/plugins/ps/functions index 9a53b57de..46a0cf74f 100755 --- a/plugins/ps/functions +++ b/plugins/ps/functions @@ -11,8 +11,7 @@ get_cmd_from_procfile() { local COMMAND verify_app_name "$APP" - [[ ! -f $DOKKU_PROCFILE ]] && return - procfile-util show --procfile "$DOKKU_PROCFILE" --process-type "$PROC_TYPE" --default-port "$PORT" + fn-procfile-get-command "$DOKKU_PROCFILE" "$PROC_TYPE" "$PORT" } print_dokku_scale_file() { @@ -31,11 +30,18 @@ extract_procfile() { verify_app_name "$APP" remove_procfile "$APP" - copy_from_image "$IMAGE" "Procfile" "$DOKKU_PROCFILE" 2>/dev/null || true - if [[ -f "$DOKKU_PROCFILE" ]]; then + fn-procfile-extract "$IMAGE" "$DOKKU_PROCFILE" +} + +fn-procfile-extract() { + declare desc="extract procfile from a specified docker image" + declare IMAGE="$1" PROCFILE_PATH="$2" + + copy_from_image "$IMAGE" "Procfile" "$PROCFILE_PATH" 2>/dev/null || true + if [[ -f "$PROCFILE_PATH" ]]; then dokku_log_info1_quiet "App Procfile file found" # shellcheck disable=SC2069 - PROCFILE_ERRORS="$(procfile-util check --procfile "$DOKKU_PROCFILE" 2>&1 >/dev/null || true)" + PROCFILE_ERRORS="$(procfile-util check --procfile "$PROCFILE_PATH" 2>&1 >/dev/null || true)" if [[ -n "$PROCFILE_ERRORS" ]]; then dokku_log_fail "${PROCFILE_ERRORS}" fi @@ -44,6 +50,17 @@ extract_procfile() { fi } +fn-procfile-get-command() { + declare desc="Fetch a command from a Procfile" + declare PROCFILE_PATH="$1" PROCESS_TYPE="$2" PORT="$3" + + if [[ ! -f "$PROCFILE_PATH" ]]; then + return + fi + + procfile-util show --procfile "$PROCFILE_PATH" --process-type "$PROCESS_TYPE" --default-port "$PORT" +} + remove_procfile() { declare desc="removes DOKKU_PROCFILE" local APP="$1" diff --git a/plugins/ps/procfile-extract b/plugins/ps/procfile-extract new file mode 100755 index 000000000..5757705b6 --- /dev/null +++ b/plugins/ps/procfile-extract @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_AVAILABLE_PATH/ps/functions" + +trigger-ps-procfile-extract() { + declare desc="ps procfile-extract plugin trigger" + declare trigger="procfile-extract" + declare desc="extract procfile from a specified docker image" + declare APP="$1" IMAGE="$2" FORCE_EXTRACT="$3" PROCFILE_PATH="$4" + + if [[ -z "$PROCFILE_PATH" ]]; then + PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" + fi + + if [[ "$FORCE_EXTRACT" == "true" ]]; then + rm "$PROCFILE_PATH" + fi + + fn-procfile-extract "$APP" "$IMAGE" "$PROCFILE_PATH" +} + +trigger-ps-procfile-extract "$@" diff --git a/plugins/ps/procfile-get-command b/plugins/ps/procfile-get-command new file mode 100755 index 000000000..635f0838f --- /dev/null +++ b/plugins/ps/procfile-get-command @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_AVAILABLE_PATH/ps/functions" + +trigger-ps-procfile-get-command() { + declare desc="ps procfile-get-command plugin trigger" + declare trigger="procfile-get-command" + declare desc="extract command from an app procfile" + declare APP="$1" PROCESS_TYPE="$2" PORT="$3" "$PROCFILE_PATH" + + if [[ -z "$PROCFILE_PATH" ]]; then + PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" + fi + + fn-procfile-get-command "$PROCFILE_PATH" "$PROCESS_TYPE" "$PORT" +} + +trigger-ps-procfile-get-command "$@" diff --git a/plugins/ps/procfile-remove b/plugins/ps/procfile-remove new file mode 100755 index 000000000..2d2f8373a --- /dev/null +++ b/plugins/ps/procfile-remove @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-ps-procfile-remove() { + declare desc="ps procfile-remove plugin trigger" + declare trigger="procfile-remove" + declare desc="remove procfile for a given app" + declare APP="$1" PROCFILE_PATH="$2" + + if [[ -z "$PROCFILE_PATH" ]]; then + PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" + fi + + rm "$PROCFILE_PATH" +} + +trigger-ps-procfile-remove "$@" From 939cb67daa65ae899a5142af3e0c5acd0dea1e9b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:24:42 -0400 Subject: [PATCH 09/43] fix: remove container after use --- tests/unit/app-json.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index d7f9a24fb..d76060475 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -28,7 +28,7 @@ teardown() { CID=$(docker ps -a -q -f "ancestor=dokku/${TEST_APP}" -f "label=dokku_phase_script=postdeploy") DOCKER_COMMIT_LABEL_ARGS=("--change" "LABEL org.label-schema.schema-version=1.0" "--change" "LABEL org.label-schema.vendor=dokku" "--change" "LABEL com.dokku.app-name=$TEST_APP") IMAGE_ID=$(docker commit "${DOCKER_COMMIT_LABEL_ARGS[@]}" $CID dokku-test/${TEST_APP}) - run /bin/bash -c "docker run -ti $IMAGE_ID ls /app/postdeploy.test" + run /bin/bash -c "docker run --rm -ti $IMAGE_ID ls /app/postdeploy.test" echo "output: $output" echo "status: $status" assert_success From 77f211c3723f0d3318fdcb4fa405c9e4d9d2484e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:25:06 -0400 Subject: [PATCH 10/43] feat: add command for logging container output --- plugins/common/log.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plugins/common/log.go b/plugins/common/log.go index 065eceb72..75a1689ab 100644 --- a/plugins/common/log.go +++ b/plugins/common/log.go @@ -3,6 +3,7 @@ package common import ( "fmt" "os" + "strings" ) // LogFail is the failure log formatter @@ -71,6 +72,20 @@ func LogVerboseQuiet(text string) { } } +// LogVerboseQuietContainerLogs is the verbose log formatter for container logs +func LogVerboseQuietContainerLogs(containerID string) { + sc := NewShellCmdWithArgs(DockerBin(), "container", "logs", containerID) + sc.ShowOutput = false + b, err := sc.CombinedOutput() + if err != nil { + LogExclaim(fmt.Sprintf("Failed to fetch container logs: %s", containerID)) + } + + for _, line := range strings.Split(string(b), "\n") { + LogVerboseQuiet(line) + } +} + // LogWarn is the warning log formatter func LogWarn(text string) { fmt.Fprintln(os.Stderr, fmt.Sprintf(" ! %s", text)) From 9bf7ddc2588950bd40f86c20c5278be282d6be22 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:35:40 -0400 Subject: [PATCH 11/43] fix: correct module name --- plugins/app-json/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-json/go.mod b/plugins/app-json/go.mod index 41b96b55e..125615f38 100644 --- a/plugins/app-json/go.mod +++ b/plugins/app-json/go.mod @@ -1,4 +1,4 @@ -module github.com/dokku/dokku/plugins/apps-json +module github.com/dokku/dokku/plugins/app-json go 1.14 From 1813317e8673557b39d09d74aa6291a0b057f7cc Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:36:06 -0400 Subject: [PATCH 12/43] fix: execute correct script --- plugins/app-json/triggers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index d802a92c2..2b3bd83b7 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -1,7 +1,7 @@ package appjson func TriggerPostDeploy(appName string, imageTag string) error { - if err := executeScript(appName, imageTag, "predeploy"); err != nil { + if err := executeScript(appName, imageTag, "postdeploy"); err != nil { return err } @@ -12,7 +12,7 @@ func TriggerPostDeploy(appName string, imageTag string) error { } func TriggerPreDeploy(appName string, imageTag string) error { - if err := executeScript(appName, imageTag, "postdeploy"); err != nil { + if err := executeScript(appName, imageTag, "predeploy"); err != nil { return err } return nil From 617ca885f18754e93e0c957b9919bd9260ccbda0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:37:30 -0400 Subject: [PATCH 13/43] fix: run release during predeploy --- plugins/app-json/triggers.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index 2b3bd83b7..6eebba709 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -4,10 +4,6 @@ func TriggerPostDeploy(appName string, imageTag string) error { if err := executeScript(appName, imageTag, "postdeploy"); err != nil { return err } - - if err := executeScript(appName, imageTag, "release"); err != nil { - return err - } return nil } @@ -15,5 +11,9 @@ func TriggerPreDeploy(appName string, imageTag string) error { if err := executeScript(appName, imageTag, "predeploy"); err != nil { return err } + + if err := executeScript(appName, imageTag, "release"); err != nil { + return err + } return nil } From 8f671b43aeec2f932e7a4969abfd39e1e8f9c23b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:37:47 -0400 Subject: [PATCH 14/43] feat: add helper functions for getting global config --- plugins/common/common.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/common/common.go b/plugins/common/common.go index c4d072c0f..5f91c1cae 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -910,3 +910,31 @@ func pruneUnusedImages(appName string) { pruneCmd.Command.Stderr = &stderr pruneCmd.Execute() } + +func ConfigGet(appName string, key string, defaultValue string) (string, error) { + b, err := PlugnTriggerOutput("config-get", []string{appName, key}...) + if err != nil { + return defaultValue, err + } + + value := defaultValue + if string(b[:]) != "" { + value = string(b[:]) + } + + return value, nil +} + +func ConfigGetGlobal(key string, defaultValue string) (string, error) { + b, err := PlugnTriggerOutput("config-get-global", []string{key}...) + if err != nil { + return defaultValue, err + } + + value := defaultValue + if string(b[:]) != "" { + value = string(b[:]) + } + + return value, nil +} From e6f2ee6499a0f91d6f625128a8caf81870c18814 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:38:31 -0400 Subject: [PATCH 15/43] fix: handle edge cases during procfile removal --- plugins/ps/procfile-remove | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/ps/procfile-remove b/plugins/ps/procfile-remove index 2d2f8373a..28b1ced70 100755 --- a/plugins/ps/procfile-remove +++ b/plugins/ps/procfile-remove @@ -12,7 +12,11 @@ trigger-ps-procfile-remove() { PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" fi - rm "$PROCFILE_PATH" + if [[ ! -f "$PROCFILE_PATH" ]]; then + return + fi + + rm -f "$PROCFILE_PATH" 2>/dev/null } trigger-ps-procfile-remove "$@" From 147abc5c6b8010e35b6e3a81ceba7f61f7a20e07 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:38:38 -0400 Subject: [PATCH 16/43] feat: add LogDebug --- plugins/common/log.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/common/log.go b/plugins/common/log.go index 75a1689ab..a1715999c 100644 --- a/plugins/common/log.go +++ b/plugins/common/log.go @@ -100,3 +100,10 @@ func LogExclaim(text string) { func LogStderr(text string) { fmt.Fprintln(os.Stderr, text) } + +// LogDebug is the debug log formatter +func LogDebug(text string) { + if os.Getenv("DOKKU_TRACE") == "" { + fmt.Fprintln(os.Stderr, fmt.Sprintf(" ? %s", text)) + } +} From 55c77773abac932e9f913324bee2b1f195db6921 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:39:29 -0400 Subject: [PATCH 17/43] fix: read input correct as string --- plugins/common/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index 5f91c1cae..c99d38f67 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -205,7 +205,7 @@ func CopyFromImage(appName string, image string, source string, destination stri // add trailing newline for certain places where file parsing depends on it b, err := sh.Command("tail", "-c1", destination).Output() - if string(b[:]) != "" { + if string(b) != "" { f, err := os.OpenFile(destination, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err From 7ee26d41f769be7b39e5a686e752c96b89cfec10 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:39:46 -0400 Subject: [PATCH 18/43] fix: correct doc --- plugins/common/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index c99d38f67..ba65b484a 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -461,7 +461,7 @@ func DockerContainerCreate(image string, containerCreateArgs []string) (string, return string(b[:]), nil } -// DockerInspect runs an inspect command with a given format against a container id +// DockerInspect runs an inspect command with a given format against a container or image ID func DockerInspect(containerOrImageID, format string) (output string, err error) { b, err := sh.Command(DockerBin(), "inspect", "--format", format, containerOrImageID).Output() if err != nil { From f2a623fde02620dc5b04b1806dad772c775449d0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:40:01 -0400 Subject: [PATCH 19/43] fix: trim whitespace to ensure container ID is valid --- plugins/common/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index ba65b484a..56d11aafd 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -458,7 +458,7 @@ func DockerContainerCreate(image string, containerCreateArgs []string) (string, return "", errors.New(strings.TrimSpace(stderr.String())) } - return string(b[:]), nil + return strings.TrimSpace(string(b[:])), nil } // DockerInspect runs an inspect command with a given format against a container or image ID From f27096f235b2bc5cf46418b9290d9a8fe54ef411 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 13:40:53 -0400 Subject: [PATCH 20/43] refactor: implement appjson in golang --- plugins/app-json/appjson.go | 270 +++++++++++++++++++++++++++++++++++- plugins/common/common.go | 34 +++-- 2 files changed, 289 insertions(+), 15 deletions(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index b632ef67e..1806222da 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -1,20 +1,71 @@ package appjson import ( + "bytes" "encoding/json" "fmt" "io/ioutil" "os" + "strings" "github.com/dokku/dokku/plugins/common" ) type AppJson struct { Scripts struct { - Dokku string `json:"dokku"` + Dokku struct { + Predeploy string `json:"predeploy"` + Postdeploy string `json:"postdeploy"` + } `json:"dokku"` } `json:"scripts"` } +func constructScript(command string, isHerokuishImage bool) string { + script := []string{"set -eo pipefail;"} + if os.Getenv("DOKKU_TRACE") == "1" { + script = append(script, "set -x;") + } + + if isHerokuishImage { + script = append(script, []string{ + "if [[ -d '/app' ]]; then", + " export HOME=/app;", + " cd $HOME;", + "fi;", + "if [[ -d '/app/.profile.d' ]]; then", + " for file in /app/.profile.d/*; do source $file; done;", + "fi;", + + "if [[ -d '/cache' ]]; then", + " rm -rf /tmp/cache ;", + " ln -sf /cache /tmp/cache;", + "fi;", + }...) + } + + if strings.HasPrefix(command, "/") { + commandBin := strings.Split(command, " ")[0] + script = append(script, []string{ + fmt.Sprintf("if [[ ! -x \"%s\" ]]; then", commandBin), + " echo specified binary is not executable;", + " exit 1;", + "fi;", + }...) + } + + script = append(script, fmt.Sprintf("%s || exit 1;", command)) + + if isHerokuishImage { + script = append(script, []string{ + "if [[ -d '/cache' ]]; then", + " rm -f /tmp/cache;", + "fi;", + }...) + } + + return strings.Join(script, " ") +} + // getPhaseScript extracts app.json from app image and returns the appropriate json key/value func getPhaseScript(appName string, image string, phase string) (string, error) { appJsonFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "getPhaseScript")) @@ -29,7 +80,7 @@ func getPhaseScript(appName string, image string, phase string) (string, error) return "", nil } - b, err := ioutil.ReadAll(appJsonFile) + b, err := ioutil.ReadFile(appJsonFile.Name()) if err != nil { return "", fmt.Errorf("Cannot read app.json file: %v", err) } @@ -39,9 +90,222 @@ func getPhaseScript(appName string, image string, phase string) (string, error) return "", fmt.Errorf("Cannot parse app.json: %v", err) } - return appJson.Scripts.Dokku, nil + if phase == "predeploy" { + return appJson.Scripts.Dokku.Predeploy, nil + } + + return appJson.Scripts.Dokku.Postdeploy, nil +} + +// getReleaseCommand extracts the release command from a given app's procfile +func getReleaseCommand(appName string, image string) string { + forceExtract := "true" + if err := common.PlugnTrigger("procfile-extract", []string{appName, image, forceExtract}...); err != nil { + return "" + } + + processType := "release" + port := "5000" + b, _ := common.PlugnTriggerOutput("procfile-get-command", []string{appName, processType, port}...) + return string(b[:]) } func executeScript(appName string, imageTag string, phase string) error { + image := common.GetDeployingAppImageName(appName, imageTag, "") + command := "" + if phase == "release" { + command = getReleaseCommand(appName, imageTag) + } else { + var err error + if command, err = getPhaseScript(appName, image, phase); err != nil { + common.LogExclaim(err.Error()) + } + } + + if command == "" { + return nil + } + + common.LogExclaim(fmt.Sprintf("Command declared for %s phase: '%s'", phase, command)) + isHerokuishImage := common.IsImageHerokuishBased(image, appName) + script := constructScript(command, isHerokuishImage) + + imageSourceType := "dockerfile" + if isHerokuishImage { + imageSourceType = "herokuish" + } + + cacheDir := fmt.Sprintf("%s/cache", common.AppRoot(appName)) + cacheHostDir := fmt.Sprintf("%s/cache", common.AppHostRoot(appName)) + if !common.DirectoryExists(cacheDir) { + os.MkdirAll(cacheDir, 0755) + } + + var dockerArgs []string + if b, err := common.PlugnTriggerSetup("docker-args-deploy", []string{appName, imageTag}...).SetInput("").Output(); err != nil { + dockerArgs = append(dockerArgs, strings.Split(string(b[:]), "\n")...) + } + + if b, err := common.PlugnTriggerSetup("docker-args-process-deploy", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err != nil { + dockerArgs = append(dockerArgs, strings.Split(string(b[:]), "\n")...) + } + + filteredArgs := []string{"restart", "cpus", "memory", "memory-swap", "memory-reservation", "gpus"} + for _, filteredArg := range filteredArgs { + // re := regexp.MustCompile("--" + filteredArg + "=[0-9A-Za-z!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~]+ ") + + var filteredDockerArgs []string + for _, dockerArg := range dockerArgs { + if !strings.HasPrefix(dockerArg, "--"+filteredArg+"=") { + filteredDockerArgs = append(filteredDockerArgs, dockerArg) + } + } + + dockerArgs = filteredDockerArgs + } + + dockerArgs = append(dockerArgs, "--label=dokku_phase_script="+phase) + dockerArgs = append(dockerArgs, "-v", cacheHostDir+":/cache") + if os.Getenv("DOKKU_TRACE") != "" { + dockerArgs = append(dockerArgs, "--env", "DOKKU_TRACE="+os.Getenv("DOKKU_TRACE")) + } + + dokkuAppShell := "/bin/bash" + if b, _ := common.PlugnTriggerOutput("config-get-global", []string{"DOKKU_APP_SHELL"}...); string(b[:]) != "" { + dokkuAppShell = string(b[:]) + } + + if b, _ := common.PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_SHELL"}...); string(b[:]) != "" { + dokkuAppShell = string(b[:]) + } + + containerCommand := []string{dokkuAppShell, "-c", script} + containerID, _ := createdContainerID(appName, dockerArgs, image, containerCommand, phase) + common.LogVerboseQuietContainerLogs(containerID) + if !waitForExecution(containerID) { + common.LogFail(fmt.Sprintf("Execution of '%s' command failed: %s", phase, command)) + } + + if phase != "predeploy" { + return nil + } + + commitArgs := []string{"container", "commit"} + if !isHerokuishImage { + dockerfileEntrypoint, _ := common.ConfigGet(appName, "DOKKU_DOCKERFILE_CMD", "") + if dockerfileEntrypoint == "" { + dockerfileEntrypoint, _ = getEntrypointFromImage(image) + } + if dockerfileEntrypoint != "" { + commitArgs = append(commitArgs, fmt.Sprintf("--change='%s'", dockerfileEntrypoint)) + } + + dockerfileCommand, _ := common.ConfigGet(appName, "DOKKU_DOCKERFILE_CMD", "") + if dockerfileCommand == "" { + dockerfileCommand, _ = getCommandFromImage(image) + } + if dockerfileCommand != "" { + commitArgs = append(commitArgs, fmt.Sprintf("--change='%s'", dockerfileCommand)) + } + } + + commitArgs = append(commitArgs, []string{ + "--change", + "LABEL org.label-schema.schema-version=1.0", + "--change", + "LABEL org.label-schema.vendor=dokku", + "--change", + fmt.Sprintf("LABEL com.dokku.app-name=%s", appName), + "--change", + fmt.Sprintf("LABEL com.dokku.%s-phase=true", phase), + }...) + commitArgs = append(commitArgs, containerID, image) + containerCommitCmd := common.NewShellCmdWithArgs( + common.DockerBin(), + commitArgs..., + ) + containerCommitCmd.ShowOutput = false + containerCommitCmd.Command.Stderr = os.Stderr + if !containerCommitCmd.Execute() { + common.LogFail(fmt.Sprintf("Commiting of '%s' to image failed: %s", phase, command)) + } return nil } + +func getEntrypointFromImage(image string) (string, error) { + entrypoint, err := common.DockerInspect(image, "{{range .Config.Entrypoint}}{{.}} {{end}}") + if err != nil { + return "", err + } + + return fmt.Sprintf("ENTRYPOINT %s", entrypoint), err +} + +func getCommandFromImage(image string) (string, error) { + command, err := common.DockerInspect(image, "{{range .Config.Cmd}}{{.}} {{end}}") + if err != nil { + return "", err + } + + return fmt.Sprintf("CMD %s", strings.Replace(command, "/bin/sh -c", "", 1)), err +} + +func waitForExecution(containerID string) bool { + containerStartCmd := common.NewShellCmdWithArgs( + common.DockerBin(), + "container", + "start", + containerID, + ) + containerStartCmd.ShowOutput = false + containerStartCmd.Command.Stderr = os.Stderr + if !containerStartCmd.Execute() { + return false + } + + containerWaitCmd := common.NewShellCmdWithArgs( + common.DockerBin(), + "container", + "wait", + containerID, + ) + + containerWaitCmd.ShowOutput = false + containerWaitCmd.Command.Stderr = os.Stderr + b, err := containerWaitCmd.Output() + if err != nil { + return false + } + + containerExitCode := string(b[:]) + return containerExitCode != "0" +} + +func createdContainerID(appName string, dockerArgs []string, image string, command []string, phase string) (string, error) { + runLabelArgs := fmt.Sprintf("--label=com.dokku.app-name=%s", appName) + + arguments := strings.Split(common.MustGetEnv("DOKKU_GLOBAL_RUN_ARGS"), " ") + arguments = append(arguments, runLabelArgs) + arguments = append(arguments, dockerArgs...) + + arguments = append([]string{"container", "create"}, arguments...) + arguments = append(arguments, image) + arguments = append(arguments, command...) + + containerCreateCmd := common.NewShellCmdWithArgs( + common.DockerBin(), + arguments..., + ) + var stderr bytes.Buffer + containerCreateCmd.ShowOutput = false + containerCreateCmd.Command.Stderr = &stderr + + b, err := containerCreateCmd.Output() + if err != nil { + return "", err + } + + containerID := strings.TrimSpace(string(b)) + err = common.PlugnTrigger("post-container-create", []string{"app", appName, containerID, phase}...) + return containerID, err +} diff --git a/plugins/common/common.go b/plugins/common/common.go index 56d11aafd..0cfb287c3 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -35,16 +35,22 @@ func NewShellCmd(command string) *ShellCmd { items := strings.Split(command, " ") cmd := items[0] args := items[1:] + return NewShellCmdWithArgs(cmd, args...) +} + +// NewShellCmd returns a new ShellCmd struct +func NewShellCmdWithArgs(cmd string, args ...string) *ShellCmd { + commandString := strings.Join(append([]string{cmd}, args...), " ") + return &ShellCmd{ Command: exec.Command(cmd, args...), - CommandString: command, + CommandString: commandString, Args: args, ShowOutput: true, } } -// Execute is a lightweight wrapper around exec.Command -func (sc *ShellCmd) Execute() bool { +func (sc *ShellCmd) Setup() { env := os.Environ() for k, v := range sc.Env { env = append(env, fmt.Sprintf("%s=%s", k, v)) @@ -54,6 +60,12 @@ func (sc *ShellCmd) Execute() bool { sc.Command.Stdout = os.Stdout sc.Command.Stderr = os.Stderr } +} + +// Execute is a lightweight wrapper around exec.Command +func (sc *ShellCmd) Execute() bool { + sc.Setup() + if err := sc.Command.Run(); err != nil { return false } @@ -62,18 +74,16 @@ func (sc *ShellCmd) Execute() bool { // Output is a lightweight wrapper around exec.Command.Output() func (sc *ShellCmd) Output() ([]byte, error) { - env := os.Environ() - for k, v := range sc.Env { - env = append(env, fmt.Sprintf("%s=%s", k, v)) - } - sc.Command.Env = env - if sc.ShowOutput { - sc.Command.Stdout = os.Stdout - sc.Command.Stderr = os.Stderr - } + sc.Setup() return sc.Command.Output() } +// CombinedOutput is a lightweight wrapper around exec.Command.CombinedOutput() +func (sc *ShellCmd) CombinedOutput() ([]byte, error) { + sc.Setup() + return sc.Command.CombinedOutput() +} + // AppRoot returns the app root path func AppRoot(appName string) string { dokkuRoot := MustGetEnv("DOKKU_ROOT") From 68debb44f77acad3f602fcd7c5719ee897f97dfa Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 14:22:47 -0400 Subject: [PATCH 21/43] refactor: use plugin trigger to delete procfile --- plugins/ps/procfile-extract | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/procfile-extract b/plugins/ps/procfile-extract index 5757705b6..ddace43cd 100755 --- a/plugins/ps/procfile-extract +++ b/plugins/ps/procfile-extract @@ -14,7 +14,7 @@ trigger-ps-procfile-extract() { fi if [[ "$FORCE_EXTRACT" == "true" ]]; then - rm "$PROCFILE_PATH" + plugn trigger procfile-remove "$APP" "$PROCFILE_PATH" fi fn-procfile-extract "$APP" "$IMAGE" "$PROCFILE_PATH" From 6df38a56d1867fc4d8e4457f5262b06ab616731b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 14:22:55 -0400 Subject: [PATCH 22/43] fix: properly set property... --- plugins/ps/procfile-get-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/procfile-get-command b/plugins/ps/procfile-get-command index 635f0838f..00c96d6a6 100755 --- a/plugins/ps/procfile-get-command +++ b/plugins/ps/procfile-get-command @@ -7,7 +7,7 @@ trigger-ps-procfile-get-command() { declare desc="ps procfile-get-command plugin trigger" declare trigger="procfile-get-command" declare desc="extract command from an app procfile" - declare APP="$1" PROCESS_TYPE="$2" PORT="$3" "$PROCFILE_PATH" + declare APP="$1" PROCESS_TYPE="$2" PORT="$3" PROCFILE_PATH="$4" if [[ -z "$PROCFILE_PATH" ]]; then PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" From beb42b53b2871a8b2e5b677b1bbad173c14aeff1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 14:51:02 -0400 Subject: [PATCH 23/43] tests: ensure the deploy succeeeds --- tests/unit/app-json.bats | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index d76060475..925f783d1 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -14,6 +14,9 @@ teardown() { @test "(app-json) app.json scripts" { deploy_app nodejs-express + echo "output: $output" + echo "status: $status" + assert_success run /bin/bash -c "dokku --rm run $TEST_APP ls /app/prebuild.test" echo "output: $output" From 8126c63d6e594a4ae238ebbe6bf504e18eb31ef3 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 14:51:25 -0400 Subject: [PATCH 24/43] fix: properly handle return from PluginTriggerOutput --- plugins/app-json/appjson.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 1806222da..b983709cb 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -107,7 +107,7 @@ func getReleaseCommand(appName string, image string) string { processType := "release" port := "5000" b, _ := common.PlugnTriggerOutput("procfile-get-command", []string{appName, processType, port}...) - return string(b[:]) + return strings.TrimSpace(string(b[:])) } func executeScript(appName string, imageTag string, phase string) error { @@ -143,11 +143,11 @@ func executeScript(appName string, imageTag string, phase string) error { var dockerArgs []string if b, err := common.PlugnTriggerSetup("docker-args-deploy", []string{appName, imageTag}...).SetInput("").Output(); err != nil { - dockerArgs = append(dockerArgs, strings.Split(string(b[:]), "\n")...) + dockerArgs = append(dockerArgs, strings.Split(strings.TrimSpace(string(b[:])), "\n")...) } if b, err := common.PlugnTriggerSetup("docker-args-process-deploy", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err != nil { - dockerArgs = append(dockerArgs, strings.Split(string(b[:]), "\n")...) + dockerArgs = append(dockerArgs, strings.Split(strings.TrimSpace(string(b[:])), "\n")...) } filteredArgs := []string{"restart", "cpus", "memory", "memory-swap", "memory-reservation", "gpus"} @@ -171,12 +171,12 @@ func executeScript(appName string, imageTag string, phase string) error { } dokkuAppShell := "/bin/bash" - if b, _ := common.PlugnTriggerOutput("config-get-global", []string{"DOKKU_APP_SHELL"}...); string(b[:]) != "" { - dokkuAppShell = string(b[:]) + if b, _ := common.PlugnTriggerOutput("config-get-global", []string{"DOKKU_APP_SHELL"}...); strings.TrimSpace(string(b[:])) != "" { + dokkuAppShell = strings.TrimSpace(string(b[:])) } - if b, _ := common.PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_SHELL"}...); string(b[:]) != "" { - dokkuAppShell = string(b[:]) + if b, _ := common.PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_SHELL"}...); strings.TrimSpace(string(b[:])) != "" { + dokkuAppShell = strings.TrimSpace(string(b[:])) } containerCommand := []string{dokkuAppShell, "-c", script} @@ -277,8 +277,9 @@ func waitForExecution(containerID string) bool { return false } - containerExitCode := string(b[:]) - return containerExitCode != "0" + containerExitCode := strings.TrimSpace(string(b[:])) + common.LogDebug(fmt.Sprintf("Container Exit code: '%v'", containerExitCode)) + return containerExitCode == "0" } func createdContainerID(appName string, dockerArgs []string, image string, command []string, phase string) (string, error) { From 31ab2f97f61f3a2665353ebdf08d39b70f1d9ef0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Aug 2020 16:05:10 -0400 Subject: [PATCH 25/43] fix: split arguments to ensure they get parsed properly --- plugins/app-json/appjson.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index b983709cb..8b24ddf5a 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -197,7 +197,7 @@ func executeScript(appName string, imageTag string, phase string) error { dockerfileEntrypoint, _ = getEntrypointFromImage(image) } if dockerfileEntrypoint != "" { - commitArgs = append(commitArgs, fmt.Sprintf("--change='%s'", dockerfileEntrypoint)) + commitArgs = append(commitArgs, "--change", dockerfileEntrypoint) } dockerfileCommand, _ := common.ConfigGet(appName, "DOKKU_DOCKERFILE_CMD", "") @@ -205,7 +205,7 @@ func executeScript(appName string, imageTag string, phase string) error { dockerfileCommand, _ = getCommandFromImage(image) } if dockerfileCommand != "" { - commitArgs = append(commitArgs, fmt.Sprintf("--change='%s'", dockerfileCommand)) + commitArgs = append(commitArgs, "--change", dockerfileCommand) } } From 53e75e7898ee28c3c26587732ff42ca5012d9cd8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 02:04:51 -0400 Subject: [PATCH 26/43] chore: rewrite log message This change makes it more clear as to what is being executed. --- plugins/app-json/appjson.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 8b24ddf5a..0f3d7f28b 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -113,10 +113,13 @@ func getReleaseCommand(appName string, image string) string { func executeScript(appName string, imageTag string, phase string) error { image := common.GetDeployingAppImageName(appName, imageTag, "") command := "" + phaseSource := "" if phase == "release" { command = getReleaseCommand(appName, imageTag) + phaseSource = "Procfile" } else { var err error + phaseSource = "app.json" if command, err = getPhaseScript(appName, image, phase); err != nil { common.LogExclaim(err.Error()) } @@ -126,7 +129,7 @@ func executeScript(appName string, imageTag string, phase string) error { return nil } - common.LogExclaim(fmt.Sprintf("Command declared for %s phase: '%s'", phase, command)) + common.LogInfo1(fmt.Sprintf("Executing %s command from %s: '%s'", phase, phaseSource, command)) isHerokuishImage := common.IsImageHerokuishBased(image, appName) script := constructScript(command, isHerokuishImage) From 23db07ed4e22aabf4d089c2351d5722bbdd442d0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 02:05:03 -0400 Subject: [PATCH 27/43] chore: remove extra LogDebug call --- plugins/app-json/appjson.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 0f3d7f28b..11f0df41b 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -281,7 +281,6 @@ func waitForExecution(containerID string) bool { } containerExitCode := strings.TrimSpace(string(b[:])) - common.LogDebug(fmt.Sprintf("Container Exit code: '%v'", containerExitCode)) return containerExitCode == "0" } From 5965b70b39eff85388ad275aa1660e1e597f685e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 02:05:21 -0400 Subject: [PATCH 28/43] fix: do not log empty messages --- plugins/common/log.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/common/log.go b/plugins/common/log.go index a1715999c..00983cb83 100644 --- a/plugins/common/log.go +++ b/plugins/common/log.go @@ -81,8 +81,10 @@ func LogVerboseQuietContainerLogs(containerID string) { LogExclaim(fmt.Sprintf("Failed to fetch container logs: %s", containerID)) } - for _, line := range strings.Split(string(b), "\n") { - LogVerboseQuiet(line) + for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") { + if line != "" { + LogVerboseQuiet(line) + } } } From 7ca4cba8ecff0cae5f776806012c475f27df80f7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 02:06:24 -0400 Subject: [PATCH 29/43] chore: drop extra quotes --- plugins/app-json/appjson.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 11f0df41b..fba7ae836 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -129,7 +129,7 @@ func executeScript(appName string, imageTag string, phase string) error { return nil } - common.LogInfo1(fmt.Sprintf("Executing %s command from %s: '%s'", phase, phaseSource, command)) + common.LogInfo1(fmt.Sprintf("Executing %s command from %s: %s", phase, phaseSource, command)) isHerokuishImage := common.IsImageHerokuishBased(image, appName) script := constructScript(command, isHerokuishImage) From 53dabf7fd0d0c11776870b86b93ffa8b50308913 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 02:31:36 -0400 Subject: [PATCH 30/43] refactor: fix handling of string andd array CMD and ENTRYPOINT values Previous versions extracted the values as strings, prefixing the values with `/bin/sh -c '$CMD'` unnecessarily. This caused issues where we would incorrectly overrride these values when an app had a predeploy script. Closes #3143 --- docs/configuration/environment-variables.md | 2 - docs/deployment/methods/buildpacks.md | 2 +- plugins/app-json/appjson.go | 44 ++++++++++++++------ plugins/builder-dockerfile/builder-build | 6 --- tests/apps/dockerfile-entrypoint/Dockerfile | 10 +++++ tests/apps/dockerfile-entrypoint/Procfile | 1 + tests/apps/dockerfile-entrypoint/app.json | 7 ++++ tests/apps/dockerfile-entrypoint/entrypoint | 4 ++ tests/apps/dockerfile-procfile/app.json | 8 ++++ tests/unit/app-json.bats | 45 ++++++++++++++++++++- 10 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 tests/apps/dockerfile-entrypoint/Dockerfile create mode 100644 tests/apps/dockerfile-entrypoint/Procfile create mode 100644 tests/apps/dockerfile-entrypoint/app.json create mode 100755 tests/apps/dockerfile-entrypoint/entrypoint create mode 100644 tests/apps/dockerfile-procfile/app.json diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index ebe7f511c..839883ee6 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -110,9 +110,7 @@ The following config variables have special meanings and can be set in a variety | `DOKKU_DISABLE_ANSI_PREFIX_REMOVAL` | none | `dokku config:set`
`/etc/environment`
`~dokku/.dokkurc`
`~dokku/.dokkurc/*` | Disables removal of the ANSI prefix during deploys. Can be used in cases where the client deployer does not understand ansi escape codes. | | `DOKKU_DISABLE_APP_AUTOCREATION` | none | `dokku config:set` | Disables automatic creation of a non-existent app on deploy. | | `DOKKU_DOCKER_STOP_TIMEOUT` | `10` | `dokku config:set` | Configurable grace period given to the `docker stop` command. If a container has not stopped by this time, a `kill -9` signal or equivalent is sent in order to force-terminate the container. Both the `ps:stop` and `apps:destroy` commands _also_ respect this value. If not specified, the docker defaults for the [docker stop command](https://docs.docker.com/engine/reference/commandline/stop/) will be used.| -| `DOKKU_DOCKERFILE_CMD` | dockerfile cmd | `dokku config:set` | | | `DOKKU_DOCKERFILE_CACHE_BUILD` | none | `dokku config:set` | | -| `DOKKU_DOCKERFILE_ENTRYPOINT` | dockerfile entrypoint | `dokku config:set` | | | `DOKKU_DOCKERFILE_PORTS` | dockerfile ports | `dokku config:set` | | | `DOKKU_DOCKERFILE_START_CMD` | none | `dokku config:set` | | | `DOKKU_PARALLEL_ARGUMENTS`. | none | `dokku config:set` | Allows passing custom arguments to parallel for `ps:*all` commands | diff --git a/docs/deployment/methods/buildpacks.md b/docs/deployment/methods/buildpacks.md index 728dd0d38..5a64f58e9 100644 --- a/docs/deployment/methods/buildpacks.md +++ b/docs/deployment/methods/buildpacks.md @@ -164,7 +164,7 @@ dokku buildpacks:report node-js-app --buildpacks-list If an application was previously deployed via Dockerfile, the following commands should be run before a buildpack deploy will succeed: ```shell -dokku config:unset --no-restart node-js-app DOKKU_DOCKERFILE_CMD DOKKU_DOCKERFILE_ENTRYPOINT DOKKU_PROXY_PORT_MAP +dokku config:unset --no-restart node-js-app DOKKU_PROXY_PORT_MAP ``` ### Using a specific buildpack version diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index fba7ae836..03be530b6 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -195,18 +195,12 @@ func executeScript(appName string, imageTag string, phase string) error { commitArgs := []string{"container", "commit"} if !isHerokuishImage { - dockerfileEntrypoint, _ := common.ConfigGet(appName, "DOKKU_DOCKERFILE_CMD", "") - if dockerfileEntrypoint == "" { - dockerfileEntrypoint, _ = getEntrypointFromImage(image) - } + dockerfileEntrypoint, _ := getEntrypointFromImage(image) if dockerfileEntrypoint != "" { commitArgs = append(commitArgs, "--change", dockerfileEntrypoint) } - dockerfileCommand, _ := common.ConfigGet(appName, "DOKKU_DOCKERFILE_CMD", "") - if dockerfileCommand == "" { - dockerfileCommand, _ = getCommandFromImage(image) - } + dockerfileCommand, _ := getCommandFromImage(image) if dockerfileCommand != "" { commitArgs = append(commitArgs, "--change", dockerfileCommand) } @@ -236,21 +230,47 @@ func executeScript(appName string, imageTag string, phase string) error { } func getEntrypointFromImage(image string) (string, error) { - entrypoint, err := common.DockerInspect(image, "{{range .Config.Entrypoint}}{{.}} {{end}}") + output, err := common.DockerInspect(image, "{{json .Config.Entrypoint}}") if err != nil { return "", err } + if output == "null" { + return "", err + } - return fmt.Sprintf("ENTRYPOINT %s", entrypoint), err + var entrypoint []string + if err = json.Unmarshal([]byte(output), &entrypoint); err != nil { + return "", err + } + + if len(entrypoint) == 3 && entrypoint[0] == "/bin/sh" && entrypoint[1] == "-c" { + return fmt.Sprintf("ENTRYPOINT %s", entrypoint[2]), nil + } + + serializedEntrypoint, err := json.Marshal(entrypoint) + return fmt.Sprintf("ENTRYPOINT %s", string(serializedEntrypoint)), err } func getCommandFromImage(image string) (string, error) { - command, err := common.DockerInspect(image, "{{range .Config.Cmd}}{{.}} {{end}}") + output, err := common.DockerInspect(image, "{{json .Config.Cmd}}") if err != nil { return "", err } + if output == "null" { + return "", err + } - return fmt.Sprintf("CMD %s", strings.Replace(command, "/bin/sh -c", "", 1)), err + var command []string + if err = json.Unmarshal([]byte(output), &command); err != nil { + return "", err + } + + if len(command) == 3 && command[0] == "/bin/sh" && command[1] == "-c" { + return fmt.Sprintf("CMD %s", command[2]), nil + } + + serializedEntrypoint, err := json.Marshal(command) + return fmt.Sprintf("CMD %s", string(serializedEntrypoint)), err } func waitForExecution(containerID string) bool { diff --git a/plugins/builder-dockerfile/builder-build b/plugins/builder-dockerfile/builder-build index 2a1cf3f53..a443da29b 100755 --- a/plugins/builder-dockerfile/builder-build +++ b/plugins/builder-dockerfile/builder-build @@ -20,12 +20,6 @@ trigger-builder-dockerfile-builder-build() { # extract first port from Dockerfile local DOCKERFILE_PORTS=$(get_dockerfile_exposed_ports Dockerfile) [[ -n "$DOCKERFILE_PORTS" ]] && config_set --no-restart "$APP" DOKKU_DOCKERFILE_PORTS="$DOCKERFILE_PORTS" - - # extract ENTRYPOINT/CMD from Dockerfile - local DOCKERFILE_ENTRYPOINT=$(extract_directive_from_dockerfile Dockerfile ENTRYPOINT) - [[ -n "$DOCKERFILE_ENTRYPOINT" ]] && config_set --no-restart "$APP" DOKKU_DOCKERFILE_ENTRYPOINT="$DOCKERFILE_ENTRYPOINT" - local DOCKERFILE_CMD=$(extract_directive_from_dockerfile Dockerfile CMD) - [[ -n "$DOCKERFILE_CMD" ]] && config_set --no-restart "$APP" DOKKU_DOCKERFILE_CMD="$DOCKERFILE_CMD" plugn trigger pre-build-dockerfile "$APP" [[ "$DOKKU_DOCKERFILE_CACHE_BUILD" == "false" ]] && DOKKU_DOCKER_BUILD_OPTS="$DOKKU_DOCKER_BUILD_OPTS --no-cache" diff --git a/tests/apps/dockerfile-entrypoint/Dockerfile b/tests/apps/dockerfile-entrypoint/Dockerfile new file mode 100644 index 000000000..18d2ddaa5 --- /dev/null +++ b/tests/apps/dockerfile-entrypoint/Dockerfile @@ -0,0 +1,10 @@ +FROM ruby:2.5.1 + +RUN mkdir /app +WORKDIR /app + +ADD Procfile Procfile +ADD entrypoint entrypoint +ADD app.json app.json + +ENTRYPOINT ["./entrypoint"] diff --git a/tests/apps/dockerfile-entrypoint/Procfile b/tests/apps/dockerfile-entrypoint/Procfile new file mode 100644 index 000000000..f296d4a33 --- /dev/null +++ b/tests/apps/dockerfile-entrypoint/Procfile @@ -0,0 +1 @@ +web: sleep 1000 diff --git a/tests/apps/dockerfile-entrypoint/app.json b/tests/apps/dockerfile-entrypoint/app.json new file mode 100644 index 000000000..16b493cb2 --- /dev/null +++ b/tests/apps/dockerfile-entrypoint/app.json @@ -0,0 +1,7 @@ +{ + "scripts": { + "dokku": { + "predeploy": "touch /app/predeploy.test" + } + } +} diff --git a/tests/apps/dockerfile-entrypoint/entrypoint b/tests/apps/dockerfile-entrypoint/entrypoint new file mode 100755 index 000000000..9c2c70694 --- /dev/null +++ b/tests/apps/dockerfile-entrypoint/entrypoint @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "entrypoint script started with arguments $@" +exec "$@" diff --git a/tests/apps/dockerfile-procfile/app.json b/tests/apps/dockerfile-procfile/app.json new file mode 100644 index 000000000..0dbdea3ec --- /dev/null +++ b/tests/apps/dockerfile-procfile/app.json @@ -0,0 +1,8 @@ +{ + "scripts": { + "dokku": { + "predeploy": "touch /app/predeploy.test", + "postdeploy": "touch /app/postdeploy.test" + } + } +} diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index 925f783d1..243f4fc83 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -18,6 +18,12 @@ teardown() { echo "status: $status" assert_success + run docker inspect "${TEST_APP}.web.1" --format "{{json .Config.Cmd}}" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '["/start","web"]' + run /bin/bash -c "dokku --rm run $TEST_APP ls /app/prebuild.test" echo "output: $output" echo "status: $status" @@ -37,10 +43,47 @@ teardown() { assert_success } - @test "(app-json) app.json scripts missing" { deploy_app nodejs-express-noappjson echo "output: $output" echo "status: $status" assert_success } + +@test "(app-json) app.json dockerfile cmd" { + deploy_app dockerfile-procfile + echo "output: $output" + echo "status: $status" + assert_success + + run docker inspect "dokku/${TEST_APP}:latest" --format "{{json .Config.Cmd}}" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '["/bin/sh","-c","npm start"]' + + run docker inspect "dokku/${TEST_APP}:latest" --format "{{json .Config.Entrypoint}}" + echo "output: $output" + echo "status: $status" + assert_success + assert_output 'null' +} + +@test "(app-json) app.json dockerfile entrypoint" { + deploy_app dockerfile-entrypoint + echo "output: $output" + echo "status: $status" + assert_success + + run docker inspect "dokku/${TEST_APP}:latest" --format "{{json .Config.Cmd}}" + echo "output: $output" + echo "status: $status" + assert_success + assert_output 'null' + + run docker inspect "dokku/${TEST_APP}:latest" --format "{{json .Config.Entrypoint}}" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '["./entrypoint"]' +} From aaf8bc811a44f3433be13bc7abacda5e8c6c53cb Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 02:33:07 -0400 Subject: [PATCH 31/43] chore: drop now unused internal-functions file --- plugins/app-json/internal-functions | 133 ---------------------------- 1 file changed, 133 deletions(-) delete mode 100755 plugins/app-json/internal-functions diff --git a/plugins/app-json/internal-functions b/plugins/app-json/internal-functions deleted file mode 100755 index e7c12e605..000000000 --- a/plugins/app-json/internal-functions +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -[[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/ps/functions" - -get_phase_script() { - declare desc="extracts app.json from app image and returns the appropriate json key/value" - declare IMAGE="$1" PHASE_SCRIPT_KEY="$2" - local APP_JSON_FILE=$(mktemp "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") - trap "rm -rf '$APP_JSON_FILE' >/dev/null" RETURN INT TERM - - copy_from_image "$IMAGE" "app.json" "$APP_JSON_FILE" 2>/dev/null || true - - if [[ ! -f "$APP_JSON_FILE" ]]; then - return 0 - fi - - get_json_value ".scripts.dokku.${PHASE_SCRIPT_KEY}" <"$APP_JSON_FILE" -} - -get_release_cmd() { - declare desc="extracts the release command from a given app's procfile" - declare APP="$1" IMAGE_TAG="$2" - - extract_procfile "$APP" "$IMAGE_TAG" >/dev/null - trap "remove_procfile $APP" RETURN INT TERM EXIT - - get_cmd_from_procfile "$APP" "release" "5000" -} - -execute_script() { - declare desc="executes appropriate phase script key from app.json" - declare APP="$1" IMAGE_TAG="$2" PHASE_SCRIPT_KEY="$3" - local IMAGE id SCRIPT_CMD - - local DOCKER_COMMIT_LABEL_ARGS=("--change" "\"LABEL org.label-schema.schema-version=1.0\"" "--change" "\"LABEL org.label-schema.vendor=dokku\"" "--change" "\"LABEL com.dokku.app-name=$APP\"") - local DOCKER_RUN_LABEL_ARGS="--label=com.dokku.app-name=$APP" - - IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") - if [[ "$PHASE_SCRIPT_KEY" == "release" ]]; then - SCRIPT_CMD=$(get_release_cmd "$APP" "$IMAGE_TAG" 2>/dev/null || true) - else - SCRIPT_CMD=$(get_phase_script "$IMAGE" "$PHASE_SCRIPT_KEY" 2>/dev/null || true) - fi - - if [[ -z "$SCRIPT_CMD" ]]; then - return - fi - - dokku_log_exclaim "${PHASE_SCRIPT_KEY^} command declared: '$SCRIPT_CMD'" - local COMMAND - COMMAND="set -eo pipefail; [[ \$DOKKU_TRACE ]] && set -x ; " - COMMAND+=" if [[ -d '/app' ]]; then " - COMMAND+=" export HOME=/app ; " - COMMAND+=" cd \$HOME ; " - COMMAND+=" fi ; " - COMMAND+=" if [[ -d '/app/.profile.d' ]]; then " - COMMAND+=" for file in /app/.profile.d/*; do source \$file; done ; " - COMMAND+=" fi ; " - COMMAND+=" if [[ -d '/cache' ]]; then " - COMMAND+=" rm -rf /tmp/cache ; " - COMMAND+=" ln -sf /cache /tmp/cache ; " - COMMAND+=" fi ; " - - if [[ "$SCRIPT_CMD" == /* ]]; then - local SCRIPT_BIN="$(echo "$SCRIPT_CMD" | cut -d' ' -f1)" - COMMAND+=" if [[ ! -x \"$SCRIPT_BIN\" ]]; then " - COMMAND+=" echo specified binary is not executable ; " - COMMAND+=" exit 1 ; " - COMMAND+=" fi ; " - fi - - COMMAND+=" $SCRIPT_CMD || exit 1;" - COMMAND+=" if [[ -d '/cache' ]]; then " - COMMAND+=" rm -f /tmp/cache ; " - COMMAND+=" fi ; " - - local CACHE_DIR="$DOKKU_ROOT/$APP/cache" - local CACHE_HOST_DIR="$DOKKU_HOST_ROOT/$APP/cache" - [[ -d $CACHE_DIR ]] || mkdir -p "$CACHE_DIR" - - local DOCKER_ARGS=$(: | plugn trigger docker-args-deploy "$APP" "$IMAGE_TAG") - local IMAGE_SOURCE_TYPE="dockerfile" - is_image_herokuish_based "$IMAGE" "$APP" && IMAGE_SOURCE_TYPE="herokuish" - DOCKER_ARGS+=$(: | plugn trigger docker-args-process-deploy "$APP" "$IMAGE_SOURCE_TYPE" "$IMAGE_TAG") - - # strip certain args from DOCKER_ARGS - local filtered_args=("restart" "cpus" "memory" "memory-swap" "memory-reservation" "gpus") - for filtered_arg in "${filtered_args[@]}"; do - DOCKER_ARGS=$(sed -e "s/--${filtered_arg}=[[:graph:]]\+[[:blank:]]\?//g" <<<"$DOCKER_ARGS") - done - - # eval args as array to respect escapes - declare -a ARG_ARRAY - eval "ARG_ARRAY=($DOCKER_ARGS)" - - local DOKKU_APP_SHELL="/bin/bash" - DOKKU_APP_SHELL="$(config_get --global DOKKU_APP_SHELL || echo "$DOKKU_APP_SHELL")" - DOKKU_APP_SHELL="$(config_get "$APP" DOKKU_APP_SHELL || echo "$DOKKU_APP_SHELL")" - [[ -z "$DOKKU_APP_SHELL" ]] && DOKKU_APP_SHELL="/bin/bash" - - local DOKKU_CONTAINER_EXIT_CODE=0 - id=$("$DOCKER_BIN" container create "${DOCKER_RUN_LABEL_ARGS[@]}" $DOKKU_GLOBAL_RUN_ARGS --env DOKKU_TRACE="$DOKKU_TRACE" --label=dokku_phase_script="${PHASE_SCRIPT_KEY}" -v "$CACHE_HOST_DIR:/cache" "${ARG_ARRAY[@]}" "$IMAGE" "$DOKKU_APP_SHELL" -c "$COMMAND") - plugn trigger post-container-create "app" "$id" "$APP" "$PHASE_SCRIPT_KEY" - "$DOCKER_BIN" container start "$id" >/dev/null || DOKKU_CONTAINER_EXIT_CODE=$? - DOKKU_CONTAINER_EXIT_CODE="$("$DOCKER_BIN" container wait "$id" 2>/dev/null || echo "$DOKKU_CONTAINER_EXIT_CODE")" - - if [[ "$DOKKU_CONTAINER_EXIT_CODE" -ne 0 ]]; then - dokku_container_log_verbose_quiet "$id" - dokku_log_fail "execution of '$SCRIPT_CMD' failed!" - fi - - dokku_container_log_verbose_quiet "$id" - if [[ "$PHASE_SCRIPT_KEY" != "predeploy" ]]; then - return - fi - - if ! is_image_herokuish_based "$IMAGE" "$APP"; then - local DOKKU_DOCKERFILE_ENTRYPOINT=$(config_get "$APP" DOKKU_DOCKERFILE_ENTRYPOINT) - local DOKKU_DOCKERFILE_CMD=$(config_get "$APP" DOKKU_DOCKERFILE_CMD) - [[ -z "$DOKKU_DOCKERFILE_ENTRYPOINT" ]] && DOKKU_DOCKERFILE_ENTRYPOINT="$(get_entrypoint_from_image "$IMAGE")" - [[ -z "$DOKKU_DOCKERFILE_CMD" ]] && DOKKU_DOCKERFILE_CMD="$(get_cmd_from_image "$IMAGE")" - - [[ -n "$DOKKU_DOCKERFILE_ENTRYPOINT" ]] && local DOCKER_COMMIT_ENTRYPOINT_CHANGE_ARG="--change='$DOKKU_DOCKERFILE_ENTRYPOINT'" - [[ -n "$DOKKU_DOCKERFILE_CMD" ]] && local DOCKER_COMMIT_CMD_CHANGE_ARG="--change='$DOKKU_DOCKERFILE_CMD'" - - local DOCKER_COMMIT_ARGS="$DOCKER_COMMIT_ENTRYPOINT_CHANGE_ARG $DOCKER_COMMIT_CMD_CHANGE_ARG" - fi - - eval "$DOCKER_BIN" container commit "${DOCKER_COMMIT_LABEL_ARGS[@]}" "${DOCKER_COMMIT_ARGS[@]}" "$id" "$IMAGE" >/dev/null -} From 4ab96325269da75c99cfe2b39c7d2c994e9635b9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 02:56:50 -0400 Subject: [PATCH 32/43] fix: handle case where there is no app.json Somehow, an empty file may get copied over... --- plugins/app-json/appjson.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 03be530b6..6ff61cc6a 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -85,6 +85,10 @@ func getPhaseScript(appName string, image string, phase string) (string, error) return "", fmt.Errorf("Cannot read app.json file: %v", err) } + if strings.TrimSpace(string(b)) == "" { + return "", nil + } + var appJson AppJson if err = json.Unmarshal(b, &appJson); err != nil { return "", fmt.Errorf("Cannot parse app.json: %v", err) From a2d4ba4689d160416ccf81c891205d53d7ce3b26 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 04:21:59 -0400 Subject: [PATCH 33/43] fix: set correct args for fn-procfile-extract --- plugins/ps/procfile-extract | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/procfile-extract b/plugins/ps/procfile-extract index ddace43cd..43003fb07 100755 --- a/plugins/ps/procfile-extract +++ b/plugins/ps/procfile-extract @@ -17,7 +17,7 @@ trigger-ps-procfile-extract() { plugn trigger procfile-remove "$APP" "$PROCFILE_PATH" fi - fn-procfile-extract "$APP" "$IMAGE" "$PROCFILE_PATH" + fn-procfile-extract "$IMAGE" "$PROCFILE_PATH" } trigger-ps-procfile-extract "$@" From 2478556df19ff8f37b112e1bb4d3a9e22c636d69 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 04:22:27 -0400 Subject: [PATCH 34/43] fix: ensure the base directory exists before extracting the file --- plugins/ps/procfile-extract | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/ps/procfile-extract b/plugins/ps/procfile-extract index 43003fb07..74a2ec218 100755 --- a/plugins/ps/procfile-extract +++ b/plugins/ps/procfile-extract @@ -11,6 +11,8 @@ trigger-ps-procfile-extract() { if [[ -z "$PROCFILE_PATH" ]]; then PROCFILE_PATH="$DOKKU_LIB_ROOT/data/ps/$APP/Procfile.extracted" + mkdir -p "${DOKKU_LIB_ROOT}/data/ps/${APP}" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/ps/${APP}" fi if [[ "$FORCE_EXTRACT" == "true" ]]; then From 9aab7dce7d5d68cf5c275252ce82da683f780219 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 04:22:40 -0400 Subject: [PATCH 35/43] feat: create ps directory on app create --- plugins/ps/post-create | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/ps/post-create b/plugins/ps/post-create index 8434d5444..14efa77a8 100755 --- a/plugins/ps/post-create +++ b/plugins/ps/post-create @@ -12,6 +12,9 @@ trigger-ps-post-create() { local passed_phases=(deploy) add_passed_docker_option passed_phases[@] "--restart=on-failure:10" + + mkdir -p "${DOKKU_LIB_ROOT}/data/ps/${APP}" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/ps/${APP}" } trigger-ps-post-create "$@" From 428beb3413ee833725aa2cd5383ea0413bac3df7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 04:23:10 -0400 Subject: [PATCH 36/43] fix: pass in correct image variable --- plugins/app-json/appjson.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 6ff61cc6a..03525c2c9 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -119,7 +119,7 @@ func executeScript(appName string, imageTag string, phase string) error { command := "" phaseSource := "" if phase == "release" { - command = getReleaseCommand(appName, imageTag) + command = getReleaseCommand(appName, image) phaseSource = "Procfile" } else { var err error From 7e7931f1ce6a2746967aae1c442031c862f0dce7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 04:52:44 -0400 Subject: [PATCH 37/43] fix: add `run` prefix to deploy_app calls --- tests/unit/app-json.bats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index 243f4fc83..82c806f6b 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -13,7 +13,7 @@ teardown() { } @test "(app-json) app.json scripts" { - deploy_app nodejs-express + run deploy_app nodejs-express echo "output: $output" echo "status: $status" assert_success @@ -44,14 +44,14 @@ teardown() { } @test "(app-json) app.json scripts missing" { - deploy_app nodejs-express-noappjson + run deploy_app nodejs-express-noappjson echo "output: $output" echo "status: $status" assert_success } @test "(app-json) app.json dockerfile cmd" { - deploy_app dockerfile-procfile + run deploy_app dockerfile-procfile echo "output: $output" echo "status: $status" assert_success @@ -70,7 +70,7 @@ teardown() { } @test "(app-json) app.json dockerfile entrypoint" { - deploy_app dockerfile-entrypoint + run deploy_app dockerfile-entrypoint echo "output: $output" echo "status: $status" assert_success From 84367e1c167cfc15985f7e365deb9e94a72a4ac4 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 15:01:14 -0400 Subject: [PATCH 38/43] fix: correctly handle injecting environment variables to deployment tasks Also test that release tasks are invoked the correct environment. Closes #3817 --- plugins/app-json/appjson.go | 42 ++++-- plugins/app-json/go.mod | 7 +- plugins/app-json/go.sum | 2 + plugins/common/log.go | 7 +- tests/apps/dockerfile-release/Dockerfile | 19 +++ tests/apps/dockerfile-release/Pipfile | 17 +++ tests/apps/dockerfile-release/Pipfile.lock | 120 ++++++++++++++++ tests/apps/dockerfile-release/Procfile | 1 + tests/apps/dockerfile-release/app/__init__.py | 0 tests/apps/dockerfile-release/app/admin.py | 3 + tests/apps/dockerfile-release/app/apps.py | 5 + .../app/migrations/__init__.py | 0 tests/apps/dockerfile-release/app/models.py | 3 + tests/apps/dockerfile-release/app/tests.py | 3 + tests/apps/dockerfile-release/app/views.py | 3 + tests/apps/dockerfile-release/manage.py | 11 ++ .../dockerfile-release/project/__init__.py | 0 tests/apps/dockerfile-release/project/asgi.py | 16 +++ .../project/settings/__init__.py | 12 ++ .../project/settings/defaults.py | 135 ++++++++++++++++++ .../project/settings/dev.py | 4 + .../project/settings/production.py | 1 + .../project/settings/staging.py | 1 + tests/apps/dockerfile-release/project/urls.py | 21 +++ tests/apps/dockerfile-release/project/wsgi.py | 16 +++ tests/unit/app-json.bats | 14 ++ 26 files changed, 449 insertions(+), 14 deletions(-) create mode 100644 tests/apps/dockerfile-release/Dockerfile create mode 100644 tests/apps/dockerfile-release/Pipfile create mode 100644 tests/apps/dockerfile-release/Pipfile.lock create mode 100644 tests/apps/dockerfile-release/Procfile create mode 100644 tests/apps/dockerfile-release/app/__init__.py create mode 100644 tests/apps/dockerfile-release/app/admin.py create mode 100644 tests/apps/dockerfile-release/app/apps.py create mode 100644 tests/apps/dockerfile-release/app/migrations/__init__.py create mode 100644 tests/apps/dockerfile-release/app/models.py create mode 100644 tests/apps/dockerfile-release/app/tests.py create mode 100644 tests/apps/dockerfile-release/app/views.py create mode 100755 tests/apps/dockerfile-release/manage.py create mode 100644 tests/apps/dockerfile-release/project/__init__.py create mode 100644 tests/apps/dockerfile-release/project/asgi.py create mode 100644 tests/apps/dockerfile-release/project/settings/__init__.py create mode 100644 tests/apps/dockerfile-release/project/settings/defaults.py create mode 100644 tests/apps/dockerfile-release/project/settings/dev.py create mode 100644 tests/apps/dockerfile-release/project/settings/production.py create mode 100644 tests/apps/dockerfile-release/project/settings/staging.py create mode 100644 tests/apps/dockerfile-release/project/urls.py create mode 100644 tests/apps/dockerfile-release/project/wsgi.py diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 03525c2c9..cc5109695 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -3,12 +3,14 @@ package appjson import ( "bytes" "encoding/json" + "errors" "fmt" "io/ioutil" "os" "strings" "github.com/dokku/dokku/plugins/common" + shellquote "github.com/kballard/go-shellquote" ) type AppJson struct { @@ -133,7 +135,7 @@ func executeScript(appName string, imageTag string, phase string) error { return nil } - common.LogInfo1(fmt.Sprintf("Executing %s command from %s: %s", phase, phaseSource, command)) + common.LogInfo1(fmt.Sprintf("Executing %s task from %s: %s", phase, phaseSource, command)) isHerokuishImage := common.IsImageHerokuishBased(image, appName) script := constructScript(command, isHerokuishImage) @@ -149,12 +151,22 @@ func executeScript(appName string, imageTag string, phase string) error { } var dockerArgs []string - if b, err := common.PlugnTriggerSetup("docker-args-deploy", []string{appName, imageTag}...).SetInput("").Output(); err != nil { - dockerArgs = append(dockerArgs, strings.Split(strings.TrimSpace(string(b[:])), "\n")...) + if b, err := common.PlugnTriggerSetup("docker-args-deploy", []string{appName, imageTag}...).SetInput("").Output(); err == nil { + words, err := shellquote.Split(strings.TrimSpace(string(b[:]))) + if err != nil { + return err + } + + dockerArgs = append(dockerArgs, words...) } - if b, err := common.PlugnTriggerSetup("docker-args-process-deploy", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err != nil { - dockerArgs = append(dockerArgs, strings.Split(strings.TrimSpace(string(b[:])), "\n")...) + if b, err := common.PlugnTriggerSetup("docker-args-process-deploy", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err == nil { + words, err := shellquote.Split(strings.TrimSpace(string(b[:]))) + if err != nil { + return err + } + + dockerArgs = append(dockerArgs, words...) } filteredArgs := []string{"restart", "cpus", "memory", "memory-swap", "memory-reservation", "gpus"} @@ -187,12 +199,22 @@ func executeScript(appName string, imageTag string, phase string) error { } containerCommand := []string{dokkuAppShell, "-c", script} - containerID, _ := createdContainerID(appName, dockerArgs, image, containerCommand, phase) - common.LogVerboseQuietContainerLogs(containerID) - if !waitForExecution(containerID) { - common.LogFail(fmt.Sprintf("Execution of '%s' command failed: %s", phase, command)) + containerID, err := createdContainerID(appName, dockerArgs, image, containerCommand, phase) + if err != nil { + common.LogFail(fmt.Sprintf("Failed to create %s execution container: %s", phase, err.Error())) } + if !waitForExecution(containerID) { + common.LogInfo2Quiet(fmt.Sprintf("Start of %s %s task (%s) output", appName, phase, containerID[0:9])) + common.LogVerboseQuietContainerLogs(containerID) + common.LogInfo2Quiet(fmt.Sprintf("End of %s %s task (%s) output", appName, phase, containerID[0:9])) + common.LogFail(fmt.Sprintf("Execution of %s task failed: %s", phase, command)) + } + + common.LogInfo2Quiet(fmt.Sprintf("Start of %s %s task (%s) output", appName, phase, containerID[0:9])) + common.LogVerboseQuietContainerLogs(containerID) + common.LogInfo2Quiet(fmt.Sprintf("End of %s %s task (%s) output", appName, phase, containerID[0:9])) + if phase != "predeploy" { return nil } @@ -329,7 +351,7 @@ func createdContainerID(appName string, dockerArgs []string, image string, comma b, err := containerCreateCmd.Output() if err != nil { - return "", err + return "", errors.New(stderr.String()) } containerID := strings.TrimSpace(string(b)) diff --git a/plugins/app-json/go.mod b/plugins/app-json/go.mod index 125615f38..388f20ed8 100644 --- a/plugins/app-json/go.mod +++ b/plugins/app-json/go.mod @@ -3,9 +3,10 @@ module github.com/dokku/dokku/plugins/app-json go 1.14 require ( - github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect - github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 - github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect + github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect + github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 + github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect ) replace github.com/dokku/dokku/plugins/common => ../common diff --git a/plugins/app-json/go.sum b/plugins/app-json/go.sum index f05068f8d..ee7942d28 100644 --- a/plugins/app-json/go.sum +++ b/plugins/app-json/go.sum @@ -2,5 +2,7 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA= github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg= github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= diff --git a/plugins/common/log.go b/plugins/common/log.go index 00983cb83..3619bbe65 100644 --- a/plugins/common/log.go +++ b/plugins/common/log.go @@ -81,7 +81,12 @@ func LogVerboseQuietContainerLogs(containerID string) { LogExclaim(fmt.Sprintf("Failed to fetch container logs: %s", containerID)) } - for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") { + output := strings.TrimSpace(string(b)) + if len(output) == 0 { + return + } + + for _, line := range strings.Split(output, "\n") { if line != "" { LogVerboseQuiet(line) } diff --git a/tests/apps/dockerfile-release/Dockerfile b/tests/apps/dockerfile-release/Dockerfile new file mode 100644 index 000000000..1c53486a3 --- /dev/null +++ b/tests/apps/dockerfile-release/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.8-buster + +RUN pip install --user pipenv +ENV PATH="${PATH}:/root/.local/bin" +ENV PIPENV_VENV_IN_PROJECT=0 + +COPY . /app + +WORKDIR /app + +RUN ls -lah + +RUN rm -rf .venv + +RUN mkdir .venv + +RUN pipenv install + +CMD pipenv run gunicorn -b 0.0.0.0:8000 --workers=3 --threads=5 project.wsgi:application diff --git a/tests/apps/dockerfile-release/Pipfile b/tests/apps/dockerfile-release/Pipfile new file mode 100644 index 000000000..26df7ab95 --- /dev/null +++ b/tests/apps/dockerfile-release/Pipfile @@ -0,0 +1,17 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +django = "*" +gunicorn = "*" +djangorestframework = "*" +whitenoise = "*" +psycopg2-binary = "*" +dj-database-url = "*" + +[requires] +python_version = "3.8" diff --git a/tests/apps/dockerfile-release/Pipfile.lock b/tests/apps/dockerfile-release/Pipfile.lock new file mode 100644 index 000000000..89bb16b72 --- /dev/null +++ b/tests/apps/dockerfile-release/Pipfile.lock @@ -0,0 +1,120 @@ +{ + "_meta": { + "hash": { + "sha256": "6e4ab3d777dd8ead8bc254ab4bb72ee80c93d1c09537b30ad23beb21c8bfdba9" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "asgiref": { + "hashes": [ + "sha256:7e06d934a7718bf3975acbf87780ba678957b87c7adc056f13b6215d610695a0", + "sha256:ea448f92fc35a0ef4b1508f53a04c4670255a3f33d22a81c8fc9c872036adbe5" + ], + "version": "==3.2.3" + }, + "dj-database-url": { + "hashes": [ + "sha256:4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163", + "sha256:851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9" + ], + "index": "pypi", + "version": "==0.5.0" + }, + "django": { + "hashes": [ + "sha256:4f2c913303be4f874015993420bf0bd8fd2097a9c88e6b49c6a92f9bdd3fb13a", + "sha256:8c3575f81e11390893860d97e1e0154c47512f180ea55bd84ce8fa69ba8051ca" + ], + "index": "pypi", + "version": "==3.0.2" + }, + "djangorestframework": { + "hashes": [ + "sha256:05809fc66e1c997fd9a32ea5730d9f4ba28b109b9da71fccfa5ff241201fd0a4", + "sha256:e782087823c47a26826ee5b6fa0c542968219263fb3976ec3c31edab23a4001f" + ], + "index": "pypi", + "version": "==3.11.0" + }, + "gunicorn": { + "hashes": [ + "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626", + "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c" + ], + "index": "pypi", + "version": "==20.0.4" + }, + "psycopg2-binary": { + "hashes": [ + "sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29", + "sha256:086f7e89ec85a6704db51f68f0dcae432eff9300809723a6e8782c41c2f48e03", + "sha256:18ca813fdb17bc1db73fe61b196b05dd1ca2165b884dd5ec5568877cabf9b039", + "sha256:19dc39616850342a2a6db70559af55b22955f86667b5f652f40c0e99253d9881", + "sha256:2166e770cb98f02ed5ee2b0b569d40db26788e0bf2ec3ae1a0d864ea6f1d8309", + "sha256:3a2522b1d9178575acee4adf8fd9f979f9c0449b00b4164bb63c3475ea6528ed", + "sha256:3aa773580f85a28ffdf6f862e59cb5a3cc7ef6885121f2de3fca8d6ada4dbf3b", + "sha256:3b5deaa3ee7180585a296af33e14c9b18c218d148e735c7accf78130765a47e3", + "sha256:407af6d7e46593415f216c7f56ba087a9a42bd6dc2ecb86028760aa45b802bd7", + "sha256:4c3c09fb674401f630626310bcaf6cd6285daf0d5e4c26d6e55ca26a2734e39b", + "sha256:4c6717962247445b4f9e21c962ea61d2e884fc17df5ddf5e35863b016f8a1f03", + "sha256:50446fae5681fc99f87e505d4e77c9407e683ab60c555ec302f9ac9bffa61103", + "sha256:5057669b6a66aa9ca118a2a860159f0ee3acf837eda937bdd2a64f3431361a2d", + "sha256:5dd90c5438b4f935c9d01fcbad3620253da89d19c1f5fca9158646407ed7df35", + "sha256:659c815b5b8e2a55193ede2795c1e2349b8011497310bb936da7d4745652823b", + "sha256:69b13fdf12878b10dc6003acc8d0abf3ad93e79813fd5f3812497c1c9fb9be49", + "sha256:7a1cb80e35e1ccea3e11a48afe65d38744a0e0bde88795cc56a4d05b6e4f9d70", + "sha256:7e6e3c52e6732c219c07bd97fff6c088f8df4dae3b79752ee3a817e6f32e177e", + "sha256:7f42a8490c4fe854325504ce7a6e4796b207960dabb2cbafe3c3959cb00d1d7e", + "sha256:84156313f258eafff716b2961644a4483a9be44a5d43551d554844d15d4d224e", + "sha256:8578d6b8192e4c805e85f187bc530d0f52ba86c39172e61cd51f68fddd648103", + "sha256:890167d5091279a27e2505ff0e1fb273f8c48c41d35c5b92adbf4af80e6b2ed6", + "sha256:98e10634792ac0e9e7a92a76b4991b44c2325d3e7798270a808407355e7bb0a1", + "sha256:9aadff9032e967865f9778485571e93908d27dab21d0fdfdec0ca779bb6f8ad9", + "sha256:9f24f383a298a0c0f9b3113b982e21751a8ecde6615494a3f1470eb4a9d70e9e", + "sha256:a73021b44813b5c84eda4a3af5826dd72356a900bac9bd9dd1f0f81ee1c22c2f", + "sha256:afd96845e12638d2c44d213d4810a08f4dc4a563f9a98204b7428e567014b1cd", + "sha256:b73ddf033d8cd4cc9dfed6324b1ad2a89ba52c410ef6877998422fcb9c23e3a8", + "sha256:b8f490f5fad1767a1331df1259763b3bad7d7af12a75b950c2843ba319b2415f", + "sha256:dbc5cd56fff1a6152ca59445178652756f4e509f672e49ccdf3d79c1043113a4", + "sha256:eac8a3499754790187bb00574ab980df13e754777d346f85e0ff6df929bcd964", + "sha256:eaed1c65f461a959284649e37b5051224f4db6ebdc84e40b5e65f2986f101a08" + ], + "index": "pypi", + "version": "==2.8.4" + }, + "pytz": { + "hashes": [ + "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", + "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + ], + "version": "==2019.3" + }, + "sqlparse": { + "hashes": [ + "sha256:40afe6b8d4b1117e7dff5504d7a8ce07d9a1b15aeeade8a2d10f130a834f8177", + "sha256:7c3dca29c022744e95b547e867cee89f4fce4373f3549ccd8797d8eb52cdb873" + ], + "version": "==0.3.0" + }, + "whitenoise": { + "hashes": [ + "sha256:0f9137f74bd95fa54329ace88d8dc695fbe895369a632e35f7a136e003e41d73", + "sha256:62556265ec1011bd87113fb81b7516f52688887b7a010ee899ff1fd18fd22700" + ], + "index": "pypi", + "version": "==5.0.1" + } + }, + "develop": {} +} diff --git a/tests/apps/dockerfile-release/Procfile b/tests/apps/dockerfile-release/Procfile new file mode 100644 index 000000000..fc5b1ca69 --- /dev/null +++ b/tests/apps/dockerfile-release/Procfile @@ -0,0 +1 @@ +release: pipenv run python manage.py migrate diff --git a/tests/apps/dockerfile-release/app/__init__.py b/tests/apps/dockerfile-release/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/dockerfile-release/app/admin.py b/tests/apps/dockerfile-release/app/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/tests/apps/dockerfile-release/app/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/tests/apps/dockerfile-release/app/apps.py b/tests/apps/dockerfile-release/app/apps.py new file mode 100644 index 000000000..4f29a567f --- /dev/null +++ b/tests/apps/dockerfile-release/app/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class AppMainConfig(AppConfig): + name = 'app' diff --git a/tests/apps/dockerfile-release/app/migrations/__init__.py b/tests/apps/dockerfile-release/app/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/dockerfile-release/app/models.py b/tests/apps/dockerfile-release/app/models.py new file mode 100644 index 000000000..71a836239 --- /dev/null +++ b/tests/apps/dockerfile-release/app/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/tests/apps/dockerfile-release/app/tests.py b/tests/apps/dockerfile-release/app/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/tests/apps/dockerfile-release/app/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/tests/apps/dockerfile-release/app/views.py b/tests/apps/dockerfile-release/app/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/tests/apps/dockerfile-release/app/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/tests/apps/dockerfile-release/manage.py b/tests/apps/dockerfile-release/manage.py new file mode 100755 index 000000000..0c5ebba3b --- /dev/null +++ b/tests/apps/dockerfile-release/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os + + +def main(): + print("SECRET_KEY: {0}".format(os.getenv('SECRET_KEY'))) + + +if __name__ == '__main__': + main() diff --git a/tests/apps/dockerfile-release/project/__init__.py b/tests/apps/dockerfile-release/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/apps/dockerfile-release/project/asgi.py b/tests/apps/dockerfile-release/project/asgi.py new file mode 100644 index 000000000..46249aa11 --- /dev/null +++ b/tests/apps/dockerfile-release/project/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for project project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') + +application = get_asgi_application() diff --git a/tests/apps/dockerfile-release/project/settings/__init__.py b/tests/apps/dockerfile-release/project/settings/__init__.py new file mode 100644 index 000000000..e7999ecf3 --- /dev/null +++ b/tests/apps/dockerfile-release/project/settings/__init__.py @@ -0,0 +1,12 @@ +import os + +environment = os.getenv('ENVIRONMENT') + +if environment == 'dev': + from .dev import * +elif environment == 'staging': + from .staging import * +elif environment == 'production': + from .staging import * +else: + raise Exception("Invalid environment") diff --git a/tests/apps/dockerfile-release/project/settings/defaults.py b/tests/apps/dockerfile-release/project/settings/defaults.py new file mode 100644 index 000000000..529aabcd1 --- /dev/null +++ b/tests/apps/dockerfile-release/project/settings/defaults.py @@ -0,0 +1,135 @@ +""" +Django settings for project project. + +Generated by 'django-admin startproject' using Django 3.0.2. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +import dj_database_url + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.getenv('SECRET_KEY') + +if not SECRET_KEY: + raise Exception("Secret Key Required") + +ALLOWED_HOSTS = [ + "*", +] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'whitenoise.runserver_nostatic', + 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'project.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'project.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases + +DATABASES = { + 'default': dj_database_url.config(conn_max_age=500, ssl_require=True) +} + +del DATABASES['default']['OPTIONS']['sslmode'] + +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_ROOT = os.path.join(PROJECT_ROOT, 'staticfiles') +STATIC_URL = '/static/' + +# Extra places for collectstatic to find static files. +STATICFILES_DIRS = [ + os.path.join(PROJECT_ROOT, 'static'), +] + +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' diff --git a/tests/apps/dockerfile-release/project/settings/dev.py b/tests/apps/dockerfile-release/project/settings/dev.py new file mode 100644 index 000000000..ece308f17 --- /dev/null +++ b/tests/apps/dockerfile-release/project/settings/dev.py @@ -0,0 +1,4 @@ +from .defaults import * + + +DEBUG = True \ No newline at end of file diff --git a/tests/apps/dockerfile-release/project/settings/production.py b/tests/apps/dockerfile-release/project/settings/production.py new file mode 100644 index 000000000..dbc6ab431 --- /dev/null +++ b/tests/apps/dockerfile-release/project/settings/production.py @@ -0,0 +1 @@ +from .defaults import * \ No newline at end of file diff --git a/tests/apps/dockerfile-release/project/settings/staging.py b/tests/apps/dockerfile-release/project/settings/staging.py new file mode 100644 index 000000000..dbc6ab431 --- /dev/null +++ b/tests/apps/dockerfile-release/project/settings/staging.py @@ -0,0 +1 @@ +from .defaults import * \ No newline at end of file diff --git a/tests/apps/dockerfile-release/project/urls.py b/tests/apps/dockerfile-release/project/urls.py new file mode 100644 index 000000000..5a0c0f185 --- /dev/null +++ b/tests/apps/dockerfile-release/project/urls.py @@ -0,0 +1,21 @@ +"""project URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/tests/apps/dockerfile-release/project/wsgi.py b/tests/apps/dockerfile-release/project/wsgi.py new file mode 100644 index 000000000..eb1e0a505 --- /dev/null +++ b/tests/apps/dockerfile-release/project/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for project project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') + +application = get_wsgi_application() diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index 82c806f6b..5fe2f1a36 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -87,3 +87,17 @@ teardown() { assert_success assert_output '["./entrypoint"]' } + +@test "(app-json) app.json dockerfile release" { + run /bin/bash -c "dokku config:set --no-restart $TEST_APP SECRET_KEY=fjdkslafjdk ENVIRONMENT=dev" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app dockerfile-release + echo "output: $output" + echo "status: $status" + assert_output_contains "Executing release task from Procfile" + assert_output_contains "SECRET_KEY: fjdkslafjdk" + assert_success +} From 7fe69fa2696464b19014455e1ef62a3218912a56 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 18:26:36 -0400 Subject: [PATCH 39/43] fix: add missing hook --- plugins/20_events/procfile-get-command | 1 + 1 file changed, 1 insertion(+) create mode 120000 plugins/20_events/procfile-get-command diff --git a/plugins/20_events/procfile-get-command b/plugins/20_events/procfile-get-command new file mode 120000 index 000000000..5178a749f --- /dev/null +++ b/plugins/20_events/procfile-get-command @@ -0,0 +1 @@ +hook \ No newline at end of file From 2b65def1236fc95095cd305713b6b2389b4c1a13 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 18:36:07 -0400 Subject: [PATCH 40/43] fix: pass command directly to entrypoint Closes #3706 --- docs/advanced-usage/deployment-tasks.md | 2 + plugins/app-json/appjson.go | 53 ++++++++++++++------- tests/apps/dockerfile-entrypoint/entrypoint | 9 +++- tests/unit/app-json.bats | 14 ++++++ 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/docs/advanced-usage/deployment-tasks.md b/docs/advanced-usage/deployment-tasks.md index 5536689cb..44de27596 100644 --- a/docs/advanced-usage/deployment-tasks.md +++ b/docs/advanced-usage/deployment-tasks.md @@ -32,6 +32,8 @@ Each "phase" has different expectations and limitations: - Priming or invalidating cache stores - Running database migrations +Additionally, if using a Dockerfile with an `ENTRYPOINT`, the deployment task is passed to that entrypoint as is. + Please keep the above in mind when utilizing deployment tasks. > To execute commands on the host during a release phase, see the [plugin creation documentation](/docs/development/plugin-creation.md) docs for more information on building your own custom plugin. diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index cc5109695..af9174995 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -22,7 +22,11 @@ type AppJson struct { } `json:"scripts"` } -func constructScript(command string, isHerokuishImage bool) string { +func constructScript(command string, shell string, isHerokuishImage bool, hasEntrypoint bool) []string { + if hasEntrypoint { + return []string{command} + } + script := []string{"set -eo pipefail;"} if os.Getenv("DOKKU_TRACE") == "1" { script = append(script, "set -x;") @@ -65,7 +69,7 @@ func constructScript(command string, isHerokuishImage bool) string { }...) } - return strings.Join(script, " ") + return []string{shell, "-c", strings.Join(script, " ")} } // getPhaseScript extracts app.json from app image and returns the appropriate json key/value @@ -106,7 +110,12 @@ func getPhaseScript(appName string, image string, phase string) (string, error) // getReleaseCommand extracts the release command from a given app's procfile func getReleaseCommand(appName string, image string) string { forceExtract := "true" - if err := common.PlugnTrigger("procfile-extract", []string{appName, image, forceExtract}...); err != nil { + + err := common.SuppressOutput(func() error { + return common.PlugnTrigger("procfile-extract", []string{appName, image, forceExtract}...) + }) + + if err != nil { return "" } @@ -116,6 +125,19 @@ func getReleaseCommand(appName string, image string) string { return strings.TrimSpace(string(b[:])) } +func getDokkuAppShell(appName string) string { + dokkuAppShell := "/bin/bash" + if b, _ := common.PlugnTriggerOutput("config-get-global", []string{"DOKKU_APP_SHELL"}...); strings.TrimSpace(string(b[:])) != "" { + dokkuAppShell = strings.TrimSpace(string(b[:])) + } + + if b, _ := common.PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_SHELL"}...); strings.TrimSpace(string(b[:])) != "" { + dokkuAppShell = strings.TrimSpace(string(b[:])) + } + + return dokkuAppShell +} + func executeScript(appName string, imageTag string, phase string) error { image := common.GetDeployingAppImageName(appName, imageTag, "") command := "" @@ -137,7 +159,16 @@ func executeScript(appName string, imageTag string, phase string) error { common.LogInfo1(fmt.Sprintf("Executing %s task from %s: %s", phase, phaseSource, command)) isHerokuishImage := common.IsImageHerokuishBased(image, appName) - script := constructScript(command, isHerokuishImage) + dockerfileEntrypoint := "" + dockerfileCommand := "" + if !isHerokuishImage { + dockerfileEntrypoint, _ = getEntrypointFromImage(image) + dockerfileCommand, _ = getCommandFromImage(image) + } + + hasEntrypoint := dockerfileEntrypoint != "" + dokkuAppShell := getDokkuAppShell(appName) + script := constructScript(command, dokkuAppShell, isHerokuishImage, hasEntrypoint) imageSourceType := "dockerfile" if isHerokuishImage { @@ -189,17 +220,7 @@ func executeScript(appName string, imageTag string, phase string) error { dockerArgs = append(dockerArgs, "--env", "DOKKU_TRACE="+os.Getenv("DOKKU_TRACE")) } - dokkuAppShell := "/bin/bash" - if b, _ := common.PlugnTriggerOutput("config-get-global", []string{"DOKKU_APP_SHELL"}...); strings.TrimSpace(string(b[:])) != "" { - dokkuAppShell = strings.TrimSpace(string(b[:])) - } - - if b, _ := common.PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_SHELL"}...); strings.TrimSpace(string(b[:])) != "" { - dokkuAppShell = strings.TrimSpace(string(b[:])) - } - - containerCommand := []string{dokkuAppShell, "-c", script} - containerID, err := createdContainerID(appName, dockerArgs, image, containerCommand, phase) + containerID, err := createdContainerID(appName, dockerArgs, image, script, phase) if err != nil { common.LogFail(fmt.Sprintf("Failed to create %s execution container: %s", phase, err.Error())) } @@ -221,12 +242,10 @@ func executeScript(appName string, imageTag string, phase string) error { commitArgs := []string{"container", "commit"} if !isHerokuishImage { - dockerfileEntrypoint, _ := getEntrypointFromImage(image) if dockerfileEntrypoint != "" { commitArgs = append(commitArgs, "--change", dockerfileEntrypoint) } - dockerfileCommand, _ := getCommandFromImage(image) if dockerfileCommand != "" { commitArgs = append(commitArgs, "--change", dockerfileCommand) } diff --git a/tests/apps/dockerfile-entrypoint/entrypoint b/tests/apps/dockerfile-entrypoint/entrypoint index 9c2c70694..1b4625f4f 100755 --- a/tests/apps/dockerfile-entrypoint/entrypoint +++ b/tests/apps/dockerfile-entrypoint/entrypoint @@ -1,4 +1,9 @@ #!/bin/bash +set -o pipefail -echo "entrypoint script started with arguments $@" -exec "$@" +if [[ "$1" == touch* ]]; then + echo "entrypoint script started with arguments $@" + touch "$(echo $@ | awk '{print $2}')" +else + exec "$@" +fi diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index 5fe2f1a36..1bd0d2f03 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -101,3 +101,17 @@ teardown() { assert_output_contains "SECRET_KEY: fjdkslafjdk" assert_success } + +@test "(app-json) app.json dockerfile entrypoint" { + run deploy_app dockerfile-entrypoint + echo "output: $output" + echo "status: $status" + assert_output_contains "Executing predeploy task from app.json" + assert_output_contains "entrypoint script started with arguments touch /app/predeploy.test" + assert_success + + run /bin/bash -c "dokku --rm run $TEST_APP ls /app/predeploy.test" + echo "output: $output" + echo "status: $status" + assert_success +} From b8b6d64f75fbe10474b4fcf17a39c08a4733afba Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 18:44:00 -0400 Subject: [PATCH 41/43] fix: correct stickler-ci errors and remove unused code --- plugins/app-json/appjson.go | 20 +++++++++--------- plugins/app-json/triggers.go | 2 ++ plugins/common/common.go | 40 ++++++------------------------------ 3 files changed, 18 insertions(+), 44 deletions(-) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index af9174995..4864615e0 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -13,7 +13,7 @@ import ( shellquote "github.com/kballard/go-shellquote" ) -type AppJson struct { +type AppJSON struct { Scripts struct { Dokku struct { Predeploy string `json:"predeploy"` @@ -74,19 +74,19 @@ func constructScript(command string, shell string, isHerokuishImage bool, hasEnt // getPhaseScript extracts app.json from app image and returns the appropriate json key/value func getPhaseScript(appName string, image string, phase string) (string, error) { - appJsonFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "getPhaseScript")) + appJSONFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "getPhaseScript")) if err != nil { return "", fmt.Errorf("Cannot create temporary file: %v", err) } - defer os.Remove(appJsonFile.Name()) + defer os.Remove(appJSONFile.Name()) - common.CopyFromImage(appName, image, "app.json", appJsonFile.Name()) - if !common.FileExists(appJsonFile.Name()) { + common.CopyFromImage(appName, image, "app.json", appJSONFile.Name()) + if !common.FileExists(appJSONFile.Name()) { return "", nil } - b, err := ioutil.ReadFile(appJsonFile.Name()) + b, err := ioutil.ReadFile(appJSONFile.Name()) if err != nil { return "", fmt.Errorf("Cannot read app.json file: %v", err) } @@ -95,16 +95,16 @@ func getPhaseScript(appName string, image string, phase string) (string, error) return "", nil } - var appJson AppJson - if err = json.Unmarshal(b, &appJson); err != nil { + var appJSON AppJSON + if err = json.Unmarshal(b, &appJSON); err != nil { return "", fmt.Errorf("Cannot parse app.json: %v", err) } if phase == "predeploy" { - return appJson.Scripts.Dokku.Predeploy, nil + return appJSON.Scripts.Dokku.Predeploy, nil } - return appJson.Scripts.Dokku.Postdeploy, nil + return appJSON.Scripts.Dokku.Postdeploy, nil } // getReleaseCommand extracts the release command from a given app's procfile diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index 6eebba709..88d74c824 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -1,5 +1,6 @@ package appjson +// TriggerPostDeploy is a trigger to execute the postdeploy deployment task func TriggerPostDeploy(appName string, imageTag string) error { if err := executeScript(appName, imageTag, "postdeploy"); err != nil { return err @@ -7,6 +8,7 @@ func TriggerPostDeploy(appName string, imageTag string) error { return nil } +// TriggerPreDeploy is a trigger to execute predeploy and release deployment tasks func TriggerPreDeploy(appName string, imageTag string) error { if err := executeScript(appName, imageTag, "predeploy"); err != nil { return err diff --git a/plugins/common/common.go b/plugins/common/common.go index 0cfb287c3..dc1e282dd 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -38,7 +38,7 @@ func NewShellCmd(command string) *ShellCmd { return NewShellCmdWithArgs(cmd, args...) } -// NewShellCmd returns a new ShellCmd struct +// NewShellCmdWithArgs returns a new ShellCmd struct func NewShellCmdWithArgs(cmd string, args ...string) *ShellCmd { commandString := strings.Join(append([]string{cmd}, args...), " ") @@ -50,7 +50,7 @@ func NewShellCmdWithArgs(cmd string, args ...string) *ShellCmd { } } -func (sc *ShellCmd) Setup() { +func (sc *ShellCmd) setup() { env := os.Environ() for k, v := range sc.Env { env = append(env, fmt.Sprintf("%s=%s", k, v)) @@ -64,7 +64,7 @@ func (sc *ShellCmd) Setup() { // Execute is a lightweight wrapper around exec.Command func (sc *ShellCmd) Execute() bool { - sc.Setup() + sc.setup() if err := sc.Command.Run(); err != nil { return false @@ -74,13 +74,13 @@ func (sc *ShellCmd) Execute() bool { // Output is a lightweight wrapper around exec.Command.Output() func (sc *ShellCmd) Output() ([]byte, error) { - sc.Setup() + sc.setup() return sc.Command.Output() } // CombinedOutput is a lightweight wrapper around exec.Command.CombinedOutput() func (sc *ShellCmd) CombinedOutput() ([]byte, error) { - sc.Setup() + sc.setup() return sc.Command.CombinedOutput() } @@ -90,7 +90,7 @@ func AppRoot(appName string) string { return fmt.Sprintf("%v/%v", dokkuRoot, appName) } -// AppRoot returns the app root path +// AppHostRoot returns the app root path func AppHostRoot(appName string) string { dokkuHostRoot := MustGetEnv("DOKKU_HOST_ROOT") return fmt.Sprintf("%v/%v", dokkuHostRoot, appName) @@ -920,31 +920,3 @@ func pruneUnusedImages(appName string) { pruneCmd.Command.Stderr = &stderr pruneCmd.Execute() } - -func ConfigGet(appName string, key string, defaultValue string) (string, error) { - b, err := PlugnTriggerOutput("config-get", []string{appName, key}...) - if err != nil { - return defaultValue, err - } - - value := defaultValue - if string(b[:]) != "" { - value = string(b[:]) - } - - return value, nil -} - -func ConfigGetGlobal(key string, defaultValue string) (string, error) { - b, err := PlugnTriggerOutput("config-get-global", []string{key}...) - if err != nil { - return defaultValue, err - } - - value := defaultValue - if string(b[:]) != "" { - value = string(b[:]) - } - - return value, nil -} From 4d735b4d88111883bec1d6f63c35b2d4881803c4 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 18:55:39 -0400 Subject: [PATCH 42/43] fix: do not output error on checking for process type existence --- plugins/ps/functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ps/functions b/plugins/ps/functions index 46a0cf74f..e1dda2a68 100755 --- a/plugins/ps/functions +++ b/plugins/ps/functions @@ -58,7 +58,7 @@ fn-procfile-get-command() { return fi - procfile-util show --procfile "$PROCFILE_PATH" --process-type "$PROCESS_TYPE" --default-port "$PORT" + procfile-util show --procfile "$PROCFILE_PATH" --process-type "$PROCESS_TYPE" --default-port "$PORT" 2>/dev/null } remove_procfile() { From 2a1d3877f166e03d86740330a7bbca3a82a3ede7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 31 Aug 2020 21:03:16 -0400 Subject: [PATCH 43/43] fix: add docblock --- plugins/app-json/appjson.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 4864615e0..05057cffa 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -13,6 +13,7 @@ import ( shellquote "github.com/kballard/go-shellquote" ) +// AppJSON is a struct that represents an app.json file as understood by Dokku type AppJSON struct { Scripts struct { Dokku struct {