Merge pull request #5559 from dokku/5556-app-json-extract

Move app.json extraction from built image artifact to source code

docs/appendices/0.30.0-migration-guide.md

@@ -1,5 +1,9 @@
 # 0.30.0 Migration Guide
 
+## Changes
+
+- The `app.json` file is now extracted from the source code instead of the built image. For deploys via `git:from-image`, the file is extracted from the built image.
+
 ## Removals
 
 - Support for [SPDY](https://en.wikipedia.org/wiki/SPDY) has been removed. No major browser supports it as of 2021. Custom `nginx.conf.sigil` templates referencing spdy-related variables will continue to build until the 1.0.0 release.

docs/development/plugin-triggers.md

@@ -221,6 +221,22 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
 # TODO
 ```
 
+### `builder-get-property`
+
+- Description: Return the value for an app's builder property
+- Invoked by:
+- Arguments: `$APP $KEY`
+- Example:
+
+```shell
+#!/usr/bin/env bash
+
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+APP="$1"; PROPERTY="$2"
+
+# TODO
+```
+
 ### `builder-image-is-cnb`
 
 - Description: Checks if an image is cnb-based

plugins/app-json/.gitignore

@@ -7,3 +7,4 @@
 /triggers
 /triggers/*
 /app-json-process-deploy-parallelism
+/core-*

plugins/app-json/Makefile

@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/report subcommands/set
-TRIGGERS = triggers/app-json-process-deploy-parallelism triggers/install triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-deploy triggers/pre-deploy triggers/report
+TRIGGERS = triggers/app-json-process-deploy-parallelism triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/post-deploy triggers/pre-deploy triggers/report
 BUILD = commands subcommands triggers
 PLUGIN_NAME = app-json
 

plugins/app-json/appjson.go

@@ -1,8 +1,6 @@
 package appjson
 
 import (
-	"path/filepath"
-
 	"github.com/dokku/dokku/plugins/common"
 )
 
@@ -48,7 +46,7 @@ func GetAppjsonDirectory(appName string) string {
 	return common.GetAppDataDirectory("app-json", appName)
 }
 
-// GetAppjsonPath returns the path to a given app's extracted app.json file
+// GetAppjsonPath returns the path to a given app's extracted app.json file for use by other plugins
 func GetAppjsonPath(appName string) string {
-	return filepath.Join(GetAppjsonDirectory(appName), "app.json")
+	return getProcessSpecificAppJSONPath(appName)
 }

plugins/app-json/functions.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -80,11 +81,11 @@ func constructScript(command string, shell string, isHerokuishImage bool, isCnbI
 }
 
 func getAppJSON(appName string) (AppJSON, error) {
-	if !common.FileExists(GetAppjsonPath(appName)) {
+	if !hasAppJSON(appName) {
 		return AppJSON{}, nil
 	}
 
-	b, err := ioutil.ReadFile(GetAppjsonPath(appName))
+	b, err := ioutil.ReadFile(getProcessSpecificAppJSONPath(appName))
 	if err != nil {
 		return AppJSON{}, fmt.Errorf("Cannot read app.json file: %v", err)
 	}
@@ -101,6 +102,21 @@ func getAppJSON(appName string) (AppJSON, error) {
 	return appJSON, nil
 }
 
+func getAppJSONPath(appName string) string {
+	directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "app-json", appName)
+	return filepath.Join(directory, "app.json")
+}
+
+func getProcessSpecificAppJSONPath(appName string) string {
+	existingAppJSON := getAppJSONPath(appName)
+	processSpecificAppJSON := fmt.Sprintf("%s.%s", existingAppJSON, os.Getenv("DOKKU_PID"))
+	if common.FileExists(processSpecificAppJSON) {
+		return processSpecificAppJSON
+	}
+
+	return existingAppJSON
+}
+
 // getPhaseScript extracts app.json from app image and returns the appropriate json key/value
 func getPhaseScript(appName string, phase string) (string, error) {
 	appJSON, err := getAppJSON(appName)
@@ -155,6 +171,19 @@ func getDokkuAppShell(appName string) string {
 	return shell
 }
 
+func hasAppJSON(appName string) bool {
+	appJSONPath := getAppJSONPath(appName)
+	if common.FileExists(fmt.Sprintf("%s.%s.missing", appJSONPath, os.Getenv("DOKKU_PID"))) {
+		return false
+	}
+
+	if common.FileExists(fmt.Sprintf("%s.%s", appJSONPath, os.Getenv("DOKKU_PID"))) {
+		return true
+	}
+
+	return common.FileExists(appJSONPath)
+}
+
 func cleanupDeploymentContainer(appName string, containerID string, phase string) error {
 	if phase != "predeploy" {
 		os.Setenv("DOKKU_SKIP_IMAGE_RETIRE", "true")
@@ -453,30 +482,6 @@ func createdContainerID(appName string, dockerArgs []string, image string, comma
 	return containerID, err
 }
 
-func refreshAppJSON(appName string, image string) error {
-	baseDirectory := common.GetDataDirectory("app-json")
-	if !common.DirectoryExists(baseDirectory) {
-		return errors.New("Run 'dokku plugin:install' to ensure the correct directories exist")
-	}
-
-	directory := common.GetAppDataDirectory("app-json", appName)
-	if !common.DirectoryExists(directory) {
-		if err := os.MkdirAll(directory, 0755); err != nil {
-			return err
-		}
-	}
-
-	appjsonPath := GetAppjsonPath(appName)
-	if common.FileExists(appjsonPath) {
-		if err := os.Remove(appjsonPath); err != nil {
-			return errors.New("Unable to remove previous app.json file")
-		}
-	}
-
-	common.CopyFromImage(appName, image, reportComputedAppjsonpath(appName), appjsonPath)
-	return nil
-}
-
 func setScale(appName string, image string) error {
 	appJSON, err := getAppJSON(appName)
 	if err != nil {

plugins/app-json/go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
+	github.com/otiai10/copy v1.9.0
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/sync v0.1.0
 )
@@ -12,7 +13,6 @@ require (
 require (
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
 	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
-	github.com/otiai10/copy v1.9.0 // indirect
 	github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect
 	golang.org/x/net v0.0.0-20220421235706-1d1ef9303861 // indirect
 	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect

plugins/app-json/src/triggers/triggers.go

@@ -22,6 +22,13 @@ func main() {
 		appName := flag.Arg(0)
 		processType := flag.Arg(1)
 		err = appjson.TriggerAppJSONProcessDeployParallelism(appName, processType)
+	case "core-post-deploy":
+		appName := flag.Arg(0)
+		err = appjson.TriggerCorePostDeploy(appName)
+	case "core-post-extract":
+		appName := flag.Arg(0)
+		sourceWorkDir := flag.Arg(1)
+		err = appjson.TriggerCorePostExtract(appName, sourceWorkDir)
 	case "install":
 		err = appjson.TriggerInstall()
 	case "post-app-clone-setup":

plugins/app-json/triggers.go

@@ -3,8 +3,12 @@ package appjson
 import (
 	"fmt"
 	"os"
+	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/otiai10/copy"
 )
 
 // TriggerAppJSONProcessDeployParallelism returns the max number of processes to deploy in parallel
@@ -33,6 +37,86 @@ func TriggerAppJSONProcessDeployParallelism(appName string, processType string)
 	return nil
 }
 
+// TriggerCorePostDeploy sets a property to
+// allow the app to be restored on boot
+func TriggerCorePostDeploy(appName string) error {
+	existingAppJSON := getAppJSONPath(appName)
+	processSpecificAppJSON := fmt.Sprintf("%s.%s", existingAppJSON, os.Getenv("DOKKU_PID"))
+	if common.FileExists(processSpecificAppJSON) {
+		if err := os.Rename(processSpecificAppJSON, existingAppJSON); err != nil {
+			return err
+		}
+	} else if common.FileExists(fmt.Sprintf("%s.missing", processSpecificAppJSON)) {
+		if err := os.Remove(fmt.Sprintf("%s.missing", processSpecificAppJSON)); err != nil {
+			return err
+		}
+
+		if common.FileExists(existingAppJSON) {
+			if err := os.Remove(existingAppJSON); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// TriggerCorePostExtract ensures that the main app.json is the one specified by app-json-path
+func TriggerCorePostExtract(appName string, sourceWorkDir string) error {
+	appJSONPath := strings.Trim(reportComputedAppjsonpath(appName), "/")
+	if appJSONPath == "" {
+		appJSONPath = "app.json"
+	}
+
+	existingAppJSON := getAppJSONPath(appName)
+	files, err := filepath.Glob(fmt.Sprintf("%s.*", existingAppJSON))
+	if err != nil {
+		return err
+	}
+	for _, f := range files {
+		if err := os.Remove(f); err != nil {
+			return err
+		}
+	}
+
+	processSpecificAppJSON := fmt.Sprintf("%s.%s", existingAppJSON, os.Getenv("DOKKU_PID"))
+	b, _ := common.PlugnTriggerOutput("git-get-property", []string{appName, "source-image"}...)
+	appSourceImage := strings.TrimSpace(string(b[:]))
+
+	b, _ = common.PlugnTriggerOutput("builder-get-property", []string{appName, "build-dir"}...)
+	buildDir := strings.TrimSpace(string(b[:]))
+
+	repoDefaultAppJSONPath := path.Join(sourceWorkDir, "app.json")
+	if appSourceImage == "" {
+		repoAppJSONPath := path.Join(sourceWorkDir, buildDir, appJSONPath)
+		if !common.FileExists(repoAppJSONPath) {
+			if appJSONPath != "app.json" && common.FileExists(repoDefaultAppJSONPath) {
+				if err := os.Remove(repoDefaultAppJSONPath); err != nil {
+					return err
+				}
+			}
+			return common.TouchFile(fmt.Sprintf("%s.missing", processSpecificAppJSON))
+		}
+
+		if err := copy.Copy(repoAppJSONPath, processSpecificAppJSON); err != nil {
+			return fmt.Errorf("Unable to extract app.json: %v", err.Error())
+		}
+
+		if appJSONPath != "app.json" {
+			if err := copy.Copy(repoAppJSONPath, repoDefaultAppJSONPath); err != nil {
+				return fmt.Errorf("Unable to move app.json into place: %v", err.Error())
+			}
+		}
+	} else {
+		if err := common.CopyFromImage(appName, appSourceImage, path.Join(buildDir, appJSONPath), processSpecificAppJSON); err != nil {
+			return common.TouchFile(fmt.Sprintf("%s.missing", processSpecificAppJSON))
+		}
+	}
+
+	// TODO: add validation to app.json file by ensuring it can be deserialized
+	return nil
+}
+
 // TriggerInstall initializes app-json directory structures
 func TriggerInstall() error {
 	if err := common.PropertySetup("app-json"); err != nil {
@@ -58,7 +142,7 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
 
 // TriggerPostAppRename removes the old app data
 func TriggerPostAppRename(oldAppName string, newAppName string) error {
-	return common.RemoveAppDataDirectory("app-json", oldAppName)
+	return common.MigrateAppDataDirectory("app-json", oldAppName, newAppName)
 }
 
 // TriggerPostAppRenameSetup renames app-json files
@@ -81,7 +165,7 @@ func TriggerPostCreate(appName string) error {
 
 // TriggerPostDelete destroys the app-json data for a given app container
 func TriggerPostDelete(appName string) error {
-	dataErr := os.RemoveAll(common.GetAppDataDirectory("app-json", appName))
+	dataErr := common.RemoveAppDataDirectory("app-json", appName)
 	propertyErr := common.PropertyDestroy("app-json", appName)
 
 	if dataErr != nil {
@@ -104,10 +188,6 @@ func TriggerPostDeploy(appName string, imageTag string) error {
 // TriggerPreDeploy is a trigger to execute predeploy and release deployment tasks
 func TriggerPreDeploy(appName string, imageTag string) error {
 	image := common.GetAppImageName(appName, imageTag, "")
-	if err := refreshAppJSON(appName, image); err != nil {
-		return err
-	}
-
 	if err := executeScript(appName, image, imageTag, "predeploy"); err != nil {
 		return err
 	}

plugins/builder/Makefile

@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/report subcommands/set
-TRIGGERS = triggers/builder-detect triggers/builder-image-is-cnb triggers/builder-image-is-herokuish triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report
+TRIGGERS = triggers/builder-detect triggers/builder-get-property triggers/builder-image-is-cnb triggers/builder-image-is-herokuish triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report
 BUILD = commands subcommands triggers
 PLUGIN_NAME = builder
 

plugins/builder/src/triggers/triggers.go

@@ -21,6 +21,10 @@ func main() {
 	case "builder-detect":
 		appName := flag.Arg(0)
 		err = builder.TriggerBuilderDetect(appName)
+	case "builder-get-property":
+		appName := flag.Arg(0)
+		property := flag.Arg(1)
+		err = builder.TriggerBuilderGetProperty(appName, property)
 	case "builder-image-is-cnb":
 		appName := flag.Arg(0)
 		image := flag.Arg(1)

plugins/builder/triggers.go

@@ -1,6 +1,7 @@
 package builder
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -26,6 +27,16 @@ func TriggerBuilderDetect(appName string) error {
 	return nil
 }
 
+// TriggerBuilderGetProperty writes the builder key to stdout for a given app container
+func TriggerBuilderGetProperty(appName string, key string) error {
+	if key != "selected" && key != "build-dir" {
+		return errors.New("Invalid logs property specified")
+	}
+
+	fmt.Println(common.PropertyGet("builder", appName, key))
+	return nil
+}
+
 // TriggerBuilderImageIsCNB prints true if an image is cnb based, false otherwise
 func TriggerBuilderImageIsCNB(appName string, image string) error {
 	if common.IsImageCnbBased(image) {

plugins/common/data.go

@@ -46,6 +46,15 @@ func GetDataDirectory(pluginName string) string {
 	return filepath.Join(MustGetEnv("DOKKU_LIB_ROOT"), "data", pluginName)
 }
 
+// MigrateAppDataDirectory migrates the data directory for one app to another
+func MigrateAppDataDirectory(pluginName string, oldAppName string, newAppName string) error {
+	if err := CloneAppData(pluginName, oldAppName, newAppName); err != nil {
+		return err
+	}
+
+	return RemoveAppDataDirectory(pluginName, oldAppName)
+}
+
 // RemoveAppDataDirectory removes the path to the data directory for the given plugin/app combination
 func RemoveAppDataDirectory(pluginName, appName string) error {
 	return os.RemoveAll(GetAppDataDirectory(pluginName, appName))

plugins/logs/triggers.go

@@ -118,7 +118,7 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
 
 // TriggerPostAppRename removes the old app data
 func TriggerPostAppRename(oldAppName string, newAppName string) error {
-	return common.RemoveAppDataDirectory("logs", oldAppName)
+	return common.MigrateAppDataDirectory("logs", oldAppName, newAppName)
 }
 
 // TriggerPostAppRenameSetup renames logs files
@@ -141,7 +141,7 @@ func TriggerPostCreate(appName string) error {
 
 // TriggerPostDelete destroys the logs property for a given app container
 func TriggerPostDelete(appName string) error {
-	dataErr := os.RemoveAll(common.GetAppDataDirectory("logs", appName))
+	dataErr := common.RemoveAppDataDirectory("logs", appName)
 	propertyErr := common.PropertyDestroy("logs", appName)
 
 	if dataErr != nil {

plugins/ps/functions.go

@@ -70,6 +70,16 @@ func getProcfilePath(appName string) string {
 	return filepath.Join(directory, "Procfile")
 }
 
+func getProcessSpecificProcfilePath(appName string) string {
+	existingProcfile := getProcfilePath(appName)
+	processSpecificProcfile := fmt.Sprintf("%s.%s", existingProcfile, os.Getenv("DOKKU_PID"))
+	if common.FileExists(processSpecificProcfile) {
+		return processSpecificProcfile
+	}
+
+	return existingProcfile
+}
+
 func getRestartPolicy(appName string) (string, error) {
 	options, err := dockeroptions.GetDockerOptionsForPhase(appName, "deploy")
 	if err != nil {
@@ -275,16 +285,6 @@ func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples
 	return nil
 }
 
-func getProcessSpecificProcfile(appName string) string {
-	existingProcfile := getProcfilePath(appName)
-	processSpecificProcfile := fmt.Sprintf("%s.%s", existingProcfile, os.Getenv("DOKKU_PID"))
-	if common.FileExists(processSpecificProcfile) {
-		return processSpecificProcfile
-	}
-
-	return existingProcfile
-}
-
 func updateScale(appName string, clearExisting bool, formationUpdates FormationSlice) error {
 	formations := FormationSlice{}
 	if !clearExisting {
@@ -302,7 +302,7 @@ func updateScale(appName string, clearExisting bool, formationUpdates FormationS
 	validProcessTypes := make(map[string]bool)
 	if hasProcfile(appName) {
 		var err error
-		validProcessTypes, err = processesInProcfile(getProcessSpecificProcfile(appName))
+		validProcessTypes, err = processesInProcfile(getProcessSpecificProcfilePath(appName))
 		if err != nil {
 			return err
 		}

plugins/ps/triggers.go

@@ -182,7 +182,7 @@ func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
 
 // TriggerPostAppRename rebuilds the renamed app
 func TriggerPostAppRename(oldAppName string, newAppName string) error {
-	if err := common.RemoveAppDataDirectory("ps", oldAppName); err != nil {
+	if err := common.MigrateAppDataDirectory("ps", oldAppName, newAppName); err != nil {
 		return err
 	}
 
@@ -261,13 +261,11 @@ func TriggerPreDeploy(appName string, imageTag string) error {
 
 // TriggerProcfileGetCommand fetches a command from the procfile
 func TriggerProcfileGetCommand(appName string, processType string, port int) error {
-	existingProcfile := getProcfilePath(appName)
-	processSpecificProcfile := fmt.Sprintf("%s.%s", existingProcfile, os.Getenv("DOKKU_PID"))
-	if common.FileExists(fmt.Sprintf("%s.missing", processSpecificProcfile)) {
+	if !hasProcfile(appName) {
 		return nil
 	}
 
-	command, err := getProcfileCommand(getProcessSpecificProcfile(appName), processType, port)
+	command, err := getProcfileCommand(getProcessSpecificProcfilePath(appName), processType, port)
 	if err != nil {
 		return err
 	}

tests/apps/python/app3.json

@@ -0,0 +1,8 @@
+{
+  "scripts": {
+    "dokku": {
+      "predeploy": "touch /app/predeploy3.test",
+      "postdeploy": "touch /app/postdeploy.test"
+    }
+  }
+}

tests/unit/app-json.bats

@@ -181,7 +181,7 @@ teardown() {
 }
 
 @test "(app-json:set)" {
-  run deploy_app python
+  run deploy_app python dokku@dokku.me:$TEST_APP copy_app_json_to_sub_app
   echo "output: $output"
   echo "status: $status"
   assert_success
@@ -219,4 +219,37 @@ teardown() {
   assert_success
   assert_output_contains "touch /app/predeploy.test"
   assert_output_contains "touch /app/predeploy2.test" 0
+
+  run /bin/bash -c "dokku builder:set $TEST_APP build-dir sub-app"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:rebuild $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "touch /app/predeploy.test" 0
+  assert_output_contains "touch /app/predeploy2.test"
+
+  run /bin/bash -c "dokku app-json:set $TEST_APP appjson-path app3.json"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:rebuild $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "touch /app/predeploy.test" 0
+  assert_output_contains "touch /app/predeploy2.test" 0
+  assert_output_contains "touch /app/predeploy3.test"
+}
+
+copy_app_json_to_sub_app() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  cp "$APP_REPO_DIR/app2.json" "$APP_REPO_DIR/sub-app/app.json"
+  cp "$APP_REPO_DIR/app3.json" "$APP_REPO_DIR/sub-app/app3.json"
 }