diff --git a/docs/appendices/0.30.0-migration-guide.md b/docs/appendices/0.30.0-migration-guide.md index e84e4c75a..3e6035fb8 100644 --- a/docs/appendices/0.30.0-migration-guide.md +++ b/docs/appendices/0.30.0-migration-guide.md @@ -1,5 +1,9 @@ # 0.30.0 Migration Guide +## Changes + +- The `app.json` file is now extracted from the source code instead of the built image. For deploys via `git:from-image`, the file is extracted from the built image. + ## Removals - Support for [SPDY](https://en.wikipedia.org/wiki/SPDY) has been removed. No major browser supports it as of 2021. Custom `nginx.conf.sigil` templates referencing spdy-related variables will continue to build until the 1.0.0 release. diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 06d8e6029..796dcae93 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -221,6 +221,22 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `builder-get-property` + +- Description: Return the value for an app's builder property +- Invoked by: +- Arguments: `$APP $KEY` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +APP="$1"; PROPERTY="$2" + +# TODO +``` + ### `builder-image-is-cnb` - Description: Checks if an image is cnb-based diff --git a/plugins/app-json/.gitignore b/plugins/app-json/.gitignore index 4979ae1ca..2ccee7dd2 100644 --- a/plugins/app-json/.gitignore +++ b/plugins/app-json/.gitignore @@ -7,3 +7,4 @@ /triggers /triggers/* /app-json-process-deploy-parallelism +/core-* diff --git a/plugins/app-json/Makefile b/plugins/app-json/Makefile index ebbcfa09c..db0133b5d 100644 --- a/plugins/app-json/Makefile +++ b/plugins/app-json/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/report subcommands/set -TRIGGERS = triggers/app-json-process-deploy-parallelism triggers/install triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-deploy triggers/pre-deploy triggers/report +TRIGGERS = triggers/app-json-process-deploy-parallelism triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-deploy triggers/pre-deploy triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = app-json diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go index 72568b95c..73aa402fe 100644 --- a/plugins/app-json/appjson.go +++ b/plugins/app-json/appjson.go @@ -1,8 +1,6 @@ package appjson import ( - "path/filepath" - "github.com/dokku/dokku/plugins/common" ) @@ -48,7 +46,7 @@ func GetAppjsonDirectory(appName string) string { return common.GetAppDataDirectory("app-json", appName) } -// GetAppjsonPath returns the path to a given app's extracted app.json file +// GetAppjsonPath returns the path to a given app's extracted app.json file for use by other plugins func GetAppjsonPath(appName string) string { - return filepath.Join(GetAppjsonDirectory(appName), "app.json") + return getProcessSpecificAppJSONPath(appName) } diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index e47470ac3..25ae4308a 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strconv" "strings" @@ -80,11 +81,11 @@ func constructScript(command string, shell string, isHerokuishImage bool, isCnbI } func getAppJSON(appName string) (AppJSON, error) { - if !common.FileExists(GetAppjsonPath(appName)) { + if !hasAppJSON(appName) { return AppJSON{}, nil } - b, err := ioutil.ReadFile(GetAppjsonPath(appName)) + b, err := ioutil.ReadFile(getProcessSpecificAppJSONPath(appName)) if err != nil { return AppJSON{}, fmt.Errorf("Cannot read app.json file: %v", err) } @@ -101,6 +102,21 @@ func getAppJSON(appName string) (AppJSON, error) { return appJSON, nil } +func getAppJSONPath(appName string) string { + directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "app-json", appName) + return filepath.Join(directory, "app.json") +} + +func getProcessSpecificAppJSONPath(appName string) string { + existingAppJSON := getAppJSONPath(appName) + processSpecificAppJSON := fmt.Sprintf("%s.%s", existingAppJSON, os.Getenv("DOKKU_PID")) + if common.FileExists(processSpecificAppJSON) { + return processSpecificAppJSON + } + + return existingAppJSON +} + // getPhaseScript extracts app.json from app image and returns the appropriate json key/value func getPhaseScript(appName string, phase string) (string, error) { appJSON, err := getAppJSON(appName) @@ -155,6 +171,19 @@ func getDokkuAppShell(appName string) string { return shell } +func hasAppJSON(appName string) bool { + appJSONPath := getAppJSONPath(appName) + if common.FileExists(fmt.Sprintf("%s.%s.missing", appJSONPath, os.Getenv("DOKKU_PID"))) { + return false + } + + if common.FileExists(fmt.Sprintf("%s.%s", appJSONPath, os.Getenv("DOKKU_PID"))) { + return true + } + + return common.FileExists(appJSONPath) +} + func cleanupDeploymentContainer(appName string, containerID string, phase string) error { if phase != "predeploy" { os.Setenv("DOKKU_SKIP_IMAGE_RETIRE", "true") @@ -453,30 +482,6 @@ func createdContainerID(appName string, dockerArgs []string, image string, comma return containerID, err } -func refreshAppJSON(appName string, image string) error { - baseDirectory := common.GetDataDirectory("app-json") - if !common.DirectoryExists(baseDirectory) { - return errors.New("Run 'dokku plugin:install' to ensure the correct directories exist") - } - - directory := common.GetAppDataDirectory("app-json", appName) - if !common.DirectoryExists(directory) { - if err := os.MkdirAll(directory, 0755); err != nil { - return err - } - } - - appjsonPath := GetAppjsonPath(appName) - if common.FileExists(appjsonPath) { - if err := os.Remove(appjsonPath); err != nil { - return errors.New("Unable to remove previous app.json file") - } - } - - common.CopyFromImage(appName, image, reportComputedAppjsonpath(appName), appjsonPath) - return nil -} - func setScale(appName string, image string) error { appJSON, err := getAppJSON(appName) if err != nil { diff --git a/plugins/app-json/go.mod b/plugins/app-json/go.mod index cf7c1ff80..ab4c8f9ce 100644 --- a/plugins/app-json/go.mod +++ b/plugins/app-json/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 + github.com/otiai10/copy v1.9.0 github.com/spf13/pflag v1.0.5 golang.org/x/sync v0.1.0 ) @@ -12,7 +13,6 @@ require ( require ( github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect - github.com/otiai10/copy v1.9.0 // indirect github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect golang.org/x/net v0.0.0-20220421235706-1d1ef9303861 // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/plugins/app-json/src/triggers/triggers.go b/plugins/app-json/src/triggers/triggers.go index f86a01d4c..1b49fdc4d 100644 --- a/plugins/app-json/src/triggers/triggers.go +++ b/plugins/app-json/src/triggers/triggers.go @@ -22,6 +22,13 @@ func main() { appName := flag.Arg(0) processType := flag.Arg(1) err = appjson.TriggerAppJSONProcessDeployParallelism(appName, processType) + case "core-post-deploy": + appName := flag.Arg(0) + err = appjson.TriggerCorePostDeploy(appName) + case "core-post-extract": + appName := flag.Arg(0) + sourceWorkDir := flag.Arg(1) + err = appjson.TriggerCorePostExtract(appName, sourceWorkDir) case "install": err = appjson.TriggerInstall() case "post-app-clone-setup": diff --git a/plugins/app-json/triggers.go b/plugins/app-json/triggers.go index 37a7fab34..b6e117162 100644 --- a/plugins/app-json/triggers.go +++ b/plugins/app-json/triggers.go @@ -3,8 +3,12 @@ package appjson import ( "fmt" "os" + "path" + "path/filepath" + "strings" "github.com/dokku/dokku/plugins/common" + "github.com/otiai10/copy" ) // TriggerAppJSONProcessDeployParallelism returns the max number of processes to deploy in parallel @@ -33,6 +37,86 @@ func TriggerAppJSONProcessDeployParallelism(appName string, processType string) return nil } +// TriggerCorePostDeploy sets a property to +// allow the app to be restored on boot +func TriggerCorePostDeploy(appName string) error { + existingAppJSON := getAppJSONPath(appName) + processSpecificAppJSON := fmt.Sprintf("%s.%s", existingAppJSON, os.Getenv("DOKKU_PID")) + if common.FileExists(processSpecificAppJSON) { + if err := os.Rename(processSpecificAppJSON, existingAppJSON); err != nil { + return err + } + } else if common.FileExists(fmt.Sprintf("%s.missing", processSpecificAppJSON)) { + if err := os.Remove(fmt.Sprintf("%s.missing", processSpecificAppJSON)); err != nil { + return err + } + + if common.FileExists(existingAppJSON) { + if err := os.Remove(existingAppJSON); err != nil { + return err + } + } + } + + return nil +} + +// TriggerCorePostExtract ensures that the main app.json is the one specified by app-json-path +func TriggerCorePostExtract(appName string, sourceWorkDir string) error { + appJSONPath := strings.Trim(reportComputedAppjsonpath(appName), "/") + if appJSONPath == "" { + appJSONPath = "app.json" + } + + existingAppJSON := getAppJSONPath(appName) + files, err := filepath.Glob(fmt.Sprintf("%s.*", existingAppJSON)) + if err != nil { + return err + } + for _, f := range files { + if err := os.Remove(f); err != nil { + return err + } + } + + processSpecificAppJSON := fmt.Sprintf("%s.%s", existingAppJSON, os.Getenv("DOKKU_PID")) + b, _ := common.PlugnTriggerOutput("git-get-property", []string{appName, "source-image"}...) + appSourceImage := strings.TrimSpace(string(b[:])) + + b, _ = common.PlugnTriggerOutput("builder-get-property", []string{appName, "build-dir"}...) + buildDir := strings.TrimSpace(string(b[:])) + + repoDefaultAppJSONPath := path.Join(sourceWorkDir, "app.json") + if appSourceImage == "" { + repoAppJSONPath := path.Join(sourceWorkDir, buildDir, appJSONPath) + if !common.FileExists(repoAppJSONPath) { + if appJSONPath != "app.json" && common.FileExists(repoDefaultAppJSONPath) { + if err := os.Remove(repoDefaultAppJSONPath); err != nil { + return err + } + } + return common.TouchFile(fmt.Sprintf("%s.missing", processSpecificAppJSON)) + } + + if err := copy.Copy(repoAppJSONPath, processSpecificAppJSON); err != nil { + return fmt.Errorf("Unable to extract app.json: %v", err.Error()) + } + + if appJSONPath != "app.json" { + if err := copy.Copy(repoAppJSONPath, repoDefaultAppJSONPath); err != nil { + return fmt.Errorf("Unable to move app.json into place: %v", err.Error()) + } + } + } else { + if err := common.CopyFromImage(appName, appSourceImage, path.Join(buildDir, appJSONPath), processSpecificAppJSON); err != nil { + return common.TouchFile(fmt.Sprintf("%s.missing", processSpecificAppJSON)) + } + } + + // TODO: add validation to app.json file by ensuring it can be deserialized + return nil +} + // TriggerInstall initializes app-json directory structures func TriggerInstall() error { if err := common.PropertySetup("app-json"); err != nil { @@ -58,7 +142,7 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error { // TriggerPostAppRename removes the old app data func TriggerPostAppRename(oldAppName string, newAppName string) error { - return common.RemoveAppDataDirectory("app-json", oldAppName) + return common.MigrateAppDataDirectory("app-json", oldAppName, newAppName) } // TriggerPostAppRenameSetup renames app-json files @@ -81,7 +165,7 @@ func TriggerPostCreate(appName string) error { // TriggerPostDelete destroys the app-json data for a given app container func TriggerPostDelete(appName string) error { - dataErr := os.RemoveAll(common.GetAppDataDirectory("app-json", appName)) + dataErr := common.RemoveAppDataDirectory("app-json", appName) propertyErr := common.PropertyDestroy("app-json", appName) if dataErr != nil { @@ -104,10 +188,6 @@ func TriggerPostDeploy(appName string, imageTag string) error { // TriggerPreDeploy is a trigger to execute predeploy and release deployment tasks func TriggerPreDeploy(appName string, imageTag string) error { image := common.GetAppImageName(appName, imageTag, "") - if err := refreshAppJSON(appName, image); err != nil { - return err - } - if err := executeScript(appName, image, imageTag, "predeploy"); err != nil { return err } diff --git a/plugins/builder/Makefile b/plugins/builder/Makefile index 2b9139e04..ff473113e 100644 --- a/plugins/builder/Makefile +++ b/plugins/builder/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/report subcommands/set -TRIGGERS = triggers/builder-detect triggers/builder-image-is-cnb triggers/builder-image-is-herokuish triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report +TRIGGERS = triggers/builder-detect triggers/builder-get-property triggers/builder-image-is-cnb triggers/builder-image-is-herokuish triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report BUILD = commands subcommands triggers PLUGIN_NAME = builder diff --git a/plugins/builder/src/triggers/triggers.go b/plugins/builder/src/triggers/triggers.go index 25fba8921..2205d6cce 100644 --- a/plugins/builder/src/triggers/triggers.go +++ b/plugins/builder/src/triggers/triggers.go @@ -21,6 +21,10 @@ func main() { case "builder-detect": appName := flag.Arg(0) err = builder.TriggerBuilderDetect(appName) + case "builder-get-property": + appName := flag.Arg(0) + property := flag.Arg(1) + err = builder.TriggerBuilderGetProperty(appName, property) case "builder-image-is-cnb": appName := flag.Arg(0) image := flag.Arg(1) diff --git a/plugins/builder/triggers.go b/plugins/builder/triggers.go index 0ce1e6454..be1bfe560 100644 --- a/plugins/builder/triggers.go +++ b/plugins/builder/triggers.go @@ -1,6 +1,7 @@ package builder import ( + "errors" "fmt" "io/ioutil" "os" @@ -26,6 +27,16 @@ func TriggerBuilderDetect(appName string) error { return nil } +// TriggerBuilderGetProperty writes the builder key to stdout for a given app container +func TriggerBuilderGetProperty(appName string, key string) error { + if key != "selected" && key != "build-dir" { + return errors.New("Invalid logs property specified") + } + + fmt.Println(common.PropertyGet("builder", appName, key)) + return nil +} + // TriggerBuilderImageIsCNB prints true if an image is cnb based, false otherwise func TriggerBuilderImageIsCNB(appName string, image string) error { if common.IsImageCnbBased(image) { diff --git a/plugins/common/data.go b/plugins/common/data.go index 409018c78..8aaf86a2a 100644 --- a/plugins/common/data.go +++ b/plugins/common/data.go @@ -46,6 +46,15 @@ func GetDataDirectory(pluginName string) string { return filepath.Join(MustGetEnv("DOKKU_LIB_ROOT"), "data", pluginName) } +// MigrateAppDataDirectory migrates the data directory for one app to another +func MigrateAppDataDirectory(pluginName string, oldAppName string, newAppName string) error { + if err := CloneAppData(pluginName, oldAppName, newAppName); err != nil { + return err + } + + return RemoveAppDataDirectory(pluginName, oldAppName) +} + // RemoveAppDataDirectory removes the path to the data directory for the given plugin/app combination func RemoveAppDataDirectory(pluginName, appName string) error { return os.RemoveAll(GetAppDataDirectory(pluginName, appName)) diff --git a/plugins/logs/triggers.go b/plugins/logs/triggers.go index 870ce29c6..649806740 100644 --- a/plugins/logs/triggers.go +++ b/plugins/logs/triggers.go @@ -118,7 +118,7 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error { // TriggerPostAppRename removes the old app data func TriggerPostAppRename(oldAppName string, newAppName string) error { - return common.RemoveAppDataDirectory("logs", oldAppName) + return common.MigrateAppDataDirectory("logs", oldAppName, newAppName) } // TriggerPostAppRenameSetup renames logs files @@ -141,7 +141,7 @@ func TriggerPostCreate(appName string) error { // TriggerPostDelete destroys the logs property for a given app container func TriggerPostDelete(appName string) error { - dataErr := os.RemoveAll(common.GetAppDataDirectory("logs", appName)) + dataErr := common.RemoveAppDataDirectory("logs", appName) propertyErr := common.PropertyDestroy("logs", appName) if dataErr != nil { diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index f7e5a7a1d..c13005cf1 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -70,6 +70,16 @@ func getProcfilePath(appName string) string { return filepath.Join(directory, "Procfile") } +func getProcessSpecificProcfilePath(appName string) string { + existingProcfile := getProcfilePath(appName) + processSpecificProcfile := fmt.Sprintf("%s.%s", existingProcfile, os.Getenv("DOKKU_PID")) + if common.FileExists(processSpecificProcfile) { + return processSpecificProcfile + } + + return existingProcfile +} + func getRestartPolicy(appName string) (string, error) { options, err := dockeroptions.GetDockerOptionsForPhase(appName, "deploy") if err != nil { @@ -275,16 +285,6 @@ func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples return nil } -func getProcessSpecificProcfile(appName string) string { - existingProcfile := getProcfilePath(appName) - processSpecificProcfile := fmt.Sprintf("%s.%s", existingProcfile, os.Getenv("DOKKU_PID")) - if common.FileExists(processSpecificProcfile) { - return processSpecificProcfile - } - - return existingProcfile -} - func updateScale(appName string, clearExisting bool, formationUpdates FormationSlice) error { formations := FormationSlice{} if !clearExisting { @@ -302,7 +302,7 @@ func updateScale(appName string, clearExisting bool, formationUpdates FormationS validProcessTypes := make(map[string]bool) if hasProcfile(appName) { var err error - validProcessTypes, err = processesInProcfile(getProcessSpecificProcfile(appName)) + validProcessTypes, err = processesInProcfile(getProcessSpecificProcfilePath(appName)) if err != nil { return err } diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 171d5ad50..160274d22 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -182,7 +182,7 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error { // TriggerPostAppRename rebuilds the renamed app func TriggerPostAppRename(oldAppName string, newAppName string) error { - if err := common.RemoveAppDataDirectory("ps", oldAppName); err != nil { + if err := common.MigrateAppDataDirectory("ps", oldAppName, newAppName); err != nil { return err } @@ -261,13 +261,11 @@ func TriggerPreDeploy(appName string, imageTag string) error { // TriggerProcfileGetCommand fetches a command from the procfile func TriggerProcfileGetCommand(appName string, processType string, port int) error { - existingProcfile := getProcfilePath(appName) - processSpecificProcfile := fmt.Sprintf("%s.%s", existingProcfile, os.Getenv("DOKKU_PID")) - if common.FileExists(fmt.Sprintf("%s.missing", processSpecificProcfile)) { + if !hasProcfile(appName) { return nil } - command, err := getProcfileCommand(getProcessSpecificProcfile(appName), processType, port) + command, err := getProcfileCommand(getProcessSpecificProcfilePath(appName), processType, port) if err != nil { return err } diff --git a/tests/apps/python/app3.json b/tests/apps/python/app3.json new file mode 100644 index 000000000..b729f3dca --- /dev/null +++ b/tests/apps/python/app3.json @@ -0,0 +1,8 @@ +{ + "scripts": { + "dokku": { + "predeploy": "touch /app/predeploy3.test", + "postdeploy": "touch /app/postdeploy.test" + } + } +} diff --git a/tests/unit/app-json.bats b/tests/unit/app-json.bats index 3ea5fb674..693d28e3b 100644 --- a/tests/unit/app-json.bats +++ b/tests/unit/app-json.bats @@ -181,7 +181,7 @@ teardown() { } @test "(app-json:set)" { - run deploy_app python + run deploy_app python dokku@dokku.me:$TEST_APP copy_app_json_to_sub_app echo "output: $output" echo "status: $status" assert_success @@ -219,4 +219,37 @@ teardown() { assert_success assert_output_contains "touch /app/predeploy.test" assert_output_contains "touch /app/predeploy2.test" 0 + + run /bin/bash -c "dokku builder:set $TEST_APP build-dir sub-app" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "touch /app/predeploy.test" 0 + assert_output_contains "touch /app/predeploy2.test" + + run /bin/bash -c "dokku app-json:set $TEST_APP appjson-path app3.json" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "touch /app/predeploy.test" 0 + assert_output_contains "touch /app/predeploy2.test" 0 + assert_output_contains "touch /app/predeploy3.test" +} + +copy_app_json_to_sub_app() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + cp "$APP_REPO_DIR/app2.json" "$APP_REPO_DIR/sub-app/app.json" + cp "$APP_REPO_DIR/app3.json" "$APP_REPO_DIR/sub-app/app3.json" }