Merge pull request #4397 from dokku/cnb-deployment-tasks

Correctly handle environment variables in deployment tasks for Cloud Native Buildpacks
This commit is contained in:
Jose Diaz-Gonzalez authored and GitHub committed 2021-02-11 07:57:55 -05:00
commit ef2682945c
15 files changed
+189 -47

No files matched your search

+15
View File
@@ -261,6 +261,21 @@ help_content
esac esac
``` ```
### `config-export`
- Description: Returns the environment variables in a specified format
- Invoked by: app-json plugin
- Arguments: `$APP $GLOBAL $MERGED $FORMAT`
- Example:
```shell
#!/usr/bin/env bash
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
# TODO
```
### `config-get` ### `config-get`
- Description: Fetches the app config value for a key - Description: Fetches the app config value for a key
+1
View File
@@ -0,0 +1 @@
hook
+26 -6
View File
@@ -16,7 +16,7 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
func constructScript(command string, shell string, isHerokuishImage bool, hasEntrypoint bool) []string { func constructScript(command string, shell string, isHerokuishImage bool, isCnbImage bool, hasEntrypoint bool) []string {
if hasEntrypoint { if hasEntrypoint {
words, err := shellquote.Split(strings.TrimSpace(command)) words, err := shellquote.Split(strings.TrimSpace(command))
if err != nil { if err != nil {
@@ -31,7 +31,7 @@ func constructScript(command string, shell string, isHerokuishImage bool, hasEnt
script = append(script, "set -x;") script = append(script, "set -x;")
} }
if isHerokuishImage { if isHerokuishImage && !isCnbImage {
script = append(script, []string{ script = append(script, []string{
"if [[ -d '/app' ]]; then", "if [[ -d '/app' ]]; then",
" export HOME=/app;", " export HOME=/app;",
@@ -60,7 +60,7 @@ func constructScript(command string, shell string, isHerokuishImage bool, hasEnt
script = append(script, fmt.Sprintf("%s || exit 1;", command)) script = append(script, fmt.Sprintf("%s || exit 1;", command))
if isHerokuishImage { if isHerokuishImage && !isCnbImage {
script = append(script, []string{ script = append(script, []string{
"if [[ -d '/cache' ]]; then", "if [[ -d '/cache' ]]; then",
" rm -f /tmp/cache;", " rm -f /tmp/cache;",
@@ -164,6 +164,7 @@ func executeScript(appName string, image string, imageTag string, phase string)
common.LogInfo1(fmt.Sprintf("Executing %s task from %s: %s", phase, phaseSource, command)) common.LogInfo1(fmt.Sprintf("Executing %s task from %s: %s", phase, phaseSource, command))
isHerokuishImage := common.IsImageHerokuishBased(image, appName) isHerokuishImage := common.IsImageHerokuishBased(image, appName)
isCnbImage := common.IsImageCnbBased(image)
dockerfileEntrypoint := "" dockerfileEntrypoint := ""
dockerfileCommand := "" dockerfileCommand := ""
if !isHerokuishImage { if !isHerokuishImage {
@@ -173,11 +174,13 @@ func executeScript(appName string, image string, imageTag string, phase string)
hasEntrypoint := dockerfileEntrypoint != "" hasEntrypoint := dockerfileEntrypoint != ""
dokkuAppShell := getDokkuAppShell(appName) dokkuAppShell := getDokkuAppShell(appName)
script := constructScript(command, dokkuAppShell, isHerokuishImage, hasEntrypoint) script := constructScript(command, dokkuAppShell, isHerokuishImage, isCnbImage, hasEntrypoint)
imageSourceType := "dockerfile" imageSourceType := "dockerfile"
if isHerokuishImage { if isHerokuishImage {
imageSourceType = "herokuish" imageSourceType = "herokuish"
} else if isCnbImage {
imageSourceType = "cnb"
} }
cacheDir := fmt.Sprintf("%s/cache", common.AppRoot(appName)) cacheDir := fmt.Sprintf("%s/cache", common.AppRoot(appName))
@@ -224,6 +227,13 @@ func executeScript(appName string, image string, imageTag string, phase string)
if os.Getenv("DOKKU_TRACE") != "" { if os.Getenv("DOKKU_TRACE") != "" {
dockerArgs = append(dockerArgs, "--env", "DOKKU_TRACE="+os.Getenv("DOKKU_TRACE")) dockerArgs = append(dockerArgs, "--env", "DOKKU_TRACE="+os.Getenv("DOKKU_TRACE"))
} }
if isCnbImage {
// TODO: handle non-linux lifecycles
// Ideally we don't have to override this but `pack` injects the web process
// as the default entrypoint, so we need to specify the launcher so the script
// runs as expected
dockerArgs = append(dockerArgs, "--entrypoint=/cnb/lifecycle/launcher")
}
containerID, err := createdContainerID(appName, dockerArgs, image, script, phase) containerID, err := createdContainerID(appName, dockerArgs, image, script, phase)
if err != nil { if err != nil {
@@ -246,7 +256,7 @@ func executeScript(appName string, image string, imageTag string, phase string)
} }
commitArgs := []string{"container", "commit"} commitArgs := []string{"container", "commit"}
if !isHerokuishImage { if !isHerokuishImage || isCnbImage {
if dockerfileEntrypoint != "" { if dockerfileEntrypoint != "" {
commitArgs = append(commitArgs, "--change", dockerfileEntrypoint) commitArgs = append(commitArgs, "--change", dockerfileEntrypoint)
} }
@@ -366,15 +376,25 @@ func createdContainerID(appName string, dockerArgs []string, image string, comma
arguments = append(arguments, image) arguments = append(arguments, image)
arguments = append(arguments, command...) arguments = append(arguments, command...)
b, err := common.PlugnTriggerOutput("config-export", []string{appName, "false", "true", "json"}...)
if err != nil {
return "", err
}
var env map[string]string
if err := json.Unmarshal(b, &env); err != nil {
return "", err
}
containerCreateCmd := common.NewShellCmdWithArgs( containerCreateCmd := common.NewShellCmdWithArgs(
common.DockerBin(), common.DockerBin(),
arguments..., arguments...,
) )
var stderr bytes.Buffer var stderr bytes.Buffer
containerCreateCmd.Env = env
containerCreateCmd.ShowOutput = false containerCreateCmd.ShowOutput = false
containerCreateCmd.Command.Stderr = &stderr containerCreateCmd.Command.Stderr = &stderr
b, err := containerCreateCmd.Output() b, err = containerCreateCmd.Output()
if err != nil { if err != nil {
return "", errors.New(stderr.String()) return "", errors.New(stderr.String())
} }
+37 -2
View File
@@ -66,7 +66,9 @@ func CopyFromImage(appName string, image string, source string, destination stri
workDir := "" workDir := ""
if !IsAbsPath(source) { if !IsAbsPath(source) {
if IsImageHerokuishBased(image, appName) { if IsImageCnbBased(image) {
workDir = "/workspace"
} else if IsImageHerokuishBased(image, appName) {
workDir = "/app" workDir = "/app"
} else { } else {
workDir, _ = DockerInspect(image, "{{.Config.WorkingDir}}") workDir, _ = DockerInspect(image, "{{.Config.WorkingDir}}")
@@ -257,9 +259,42 @@ func DockerInspect(containerOrImageID, format string) (output string, err error)
return return
} }
// IsImageCnbBased returns true if app image is based on cnb
func IsImageCnbBased(image string) bool {
if len(image) == 0 {
return false
}
output, err := DockerInspect(image, "{{index .Config.Labels \"io.buildpacks.stack.id\" }}")
if err != nil {
return false
}
return output != ""
}
// IsImageHerokuishBased returns true if app image is based on herokuish // IsImageHerokuishBased returns true if app image is based on herokuish
func IsImageHerokuishBased(image string, appName string) bool { func IsImageHerokuishBased(image string, appName string) bool {
output, err := DockerInspect(image, "{{range .Config.Env}}{{if eq . \"USER=herokuishuser\" }}{{println .}}{{end}}{{end}}") if len(image) == 0 {
return false
}
if IsImageCnbBased(image) {
return true
}
dokkuAppUser := ""
if len(appName) != 0 {
b, err := PlugnTriggerOutput("config-get", []string{appName, "DOKKU_APP_USER"}...)
if err == nil {
dokkuAppUser = strings.TrimSpace(string(b))
}
}
if len(dokkuAppUser) == 0 {
dokkuAppUser = "herokuishuser"
}
output, err := DockerInspect(image, fmt.Sprintf("{{range .Config.Env}}{{if eq . \"USER=%s\" }}{{println .}}{{end}}{{end}}", dokkuAppUser))
if err != nil { if err != nil {
return false return false
} }
+3 -3
View File
@@ -477,10 +477,10 @@ copy_from_image() {
if verify_image "$IMAGE"; then if verify_image "$IMAGE"; then
if ! is_abs_path "$SRC_FILE"; then if ! is_abs_path "$SRC_FILE"; then
if is_image_herokuish_based "$IMAGE" "$APP"; then if is_image_cnb_based "$IMAGE"; then
WORKDIR="/app"
elif is_image_cnb_based "$IMAGE"; then
WORKDIR="/workspace" WORKDIR="/workspace"
elif is_image_herokuish_based "$IMAGE" "$APP"; then
WORKDIR="/app"
else else
WORKDIR="$("$DOCKER_BIN" image inspect --format '{{.Config.WorkingDir}}' "$IMAGE")" WORKDIR="$("$DOCKER_BIN" image inspect --format '{{.Config.WorkingDir}}' "$IMAGE")"
fi fi
+1 -1
View File
@@ -1,5 +1,5 @@
SUBCOMMANDS = subcommands/bundle subcommands/clear subcommands/export subcommands/get subcommands/keys subcommands/show subcommands/set subcommands/unset SUBCOMMANDS = subcommands/bundle subcommands/clear subcommands/export subcommands/get subcommands/keys subcommands/show subcommands/set subcommands/unset
TRIGGERS = triggers/config-get triggers/config-get-global TRIGGERS = triggers/config-export triggers/config-get triggers/config-get-global
BUILD = commands subcommands triggers BUILD = commands subcommands triggers
PLUGIN_NAME = config PLUGIN_NAME = config
+38
View File
@@ -0,0 +1,38 @@
package config
import "fmt"
func export(appName string, global bool, merged bool, format string) error {
appName, err := getAppNameOrGlobal(appName, global)
if err != nil {
return err
}
env := getEnvironment(appName, merged)
exportType := ExportFormatExports
suffix := "\n"
exportTypes := map[string]ExportFormat{
"exports": ExportFormatExports,
"envfile": ExportFormatEnvfile,
"docker-args": ExportFormatDockerArgs,
"docker-args-keys": ExportFormatDockerArgsKeys,
"shell": ExportFormatShell,
"pretty": ExportFormatPretty,
"json": ExportFormatJSON,
"json-list": ExportFormatJSONList,
}
exportType, ok := exportTypes[format]
if !ok {
return fmt.Errorf("Unknown export format: %v", format)
}
if exportType == ExportFormatShell {
suffix = " "
}
exported := env.Export(exportType)
fmt.Print(exported + suffix)
return nil
}
+6
View File
@@ -18,6 +18,12 @@ func main() {
var err error var err error
switch trigger { switch trigger {
case "config-export":
appName := flag.Arg(0)
global := flag.Arg(1)
merged := flag.Arg(2)
format := flag.Arg(3)
config.TriggerConfigExport(appName, global, merged, format)
case "config-get": case "config-get":
appName := flag.Arg(0) appName := flag.Arg(0)
key := flag.Arg(1) key := flag.Arg(1)
+1 -32
View File
@@ -33,38 +33,7 @@ func CommandClear(appName string, global bool, noRestart bool) error {
// CommandExport implements config:export // CommandExport implements config:export
func CommandExport(appName string, global bool, merged bool, format string) error { func CommandExport(appName string, global bool, merged bool, format string) error {
appName, err := getAppNameOrGlobal(appName, global) return export(appName, global, merged, format)
if err != nil {
return err
}
env := getEnvironment(appName, merged)
exportType := ExportFormatExports
suffix := "\n"
exportTypes := map[string]ExportFormat{
"exports": ExportFormatExports,
"envfile": ExportFormatEnvfile,
"docker-args": ExportFormatDockerArgs,
"docker-args-keys": ExportFormatDockerArgsKeys,
"shell": ExportFormatShell,
"pretty": ExportFormatPretty,
"json": ExportFormatJSON,
"json-list": ExportFormatJSONList,
}
exportType, ok := exportTypes[format]
if !ok {
return fmt.Errorf("Unknown export format: %v", format)
}
if exportType == ExportFormatShell {
suffix = " "
}
exported := env.Export(exportType)
fmt.Print(exported + suffix)
return nil
} }
// CommandGet implements config:get // CommandGet implements config:get
+18 -1
View File
@@ -1,6 +1,23 @@
package config package config
import "fmt" import (
"fmt"
"strconv"
)
// TriggerConfigExport returns a global config value by key
func TriggerConfigExport(appName string, global string, merged string, format string) error {
g, err := strconv.ParseBool(global)
if err != nil {
return err
}
m, err := strconv.ParseBool(merged)
if err != nil {
return err
}
return export(appName, g, m, format)
}
// TriggerConfigGet returns an app config value by key // TriggerConfigGet returns an app config value by key
func TriggerConfigGet(appName string, key string) error { func TriggerConfigGet(appName string, key string) error {
@@ -20,12 +20,15 @@ trigger-scheduler-docker-local-scheduler-deploy() {
local DOCKER_RUN_LABEL_ARGS="--label=com.dokku.app-name=$APP --label=com.dokku.container-type=deploy" local DOCKER_RUN_LABEL_ARGS="--label=com.dokku.app-name=$APP --label=com.dokku.container-type=deploy"
local DOKKU_DOCKER_STOP_TIMEOUT DOKKU_HEROKUISH DOKKU_NETWORK_BIND_ALL IMAGE local DOKKU_DOCKER_STOP_TIMEOUT DOKKU_HEROKUISH DOKKU_NETWORK_BIND_ALL IMAGE
DOKKU_HEROKUISH=false DOKKU_HEROKUISH=false
DOKKU_CNB=false
IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")
plugn trigger pre-deploy "$APP" "$IMAGE_TAG" plugn trigger pre-deploy "$APP" "$IMAGE_TAG"
is_image_cnb_based "$IMAGE" && DOKKU_CNB=true
is_image_herokuish_based "$IMAGE" "$APP" && DOKKU_HEROKUISH=true is_image_herokuish_based "$IMAGE" "$APP" && DOKKU_HEROKUISH=true
local IMAGE_SOURCE_TYPE="dockerfile" local IMAGE_SOURCE_TYPE="dockerfile"
[[ "$DOKKU_HEROKUISH" == "true" ]] && IMAGE_SOURCE_TYPE="herokuish" [[ "$DOKKU_HEROKUISH" == "true" ]] && IMAGE_SOURCE_TYPE="herokuish"
[[ "$DOKKU_CNB" == "true" ]] && IMAGE_SOURCE_TYPE="cnb"
local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE"
local oldids=$(get_app_container_ids "$APP") local oldids=$(get_app_container_ids "$APP")
@@ -81,6 +84,7 @@ trigger-scheduler-docker-local-scheduler-deploy() {
local START_CMD local START_CMD
[[ "$DOKKU_HEROKUISH" == "true" ]] && START_CMD="/start $PROC_TYPE" [[ "$DOKKU_HEROKUISH" == "true" ]] && START_CMD="/start $PROC_TYPE"
[[ "$DOKKU_CNB" == "true" ]] && START_CMD=""
[[ -n "$DOKKU_START_CMD" ]] && START_CMD="$DOKKU_START_CMD" [[ -n "$DOKKU_START_CMD" ]] && START_CMD="$DOKKU_START_CMD"
local DOKKU_PORT="" local DOKKU_PORT=""
@@ -205,7 +209,7 @@ trigger-scheduler-docker-local-scheduler-deploy() {
fn-scheduler-docker-local-extract-start-cmd() { fn-scheduler-docker-local-extract-start-cmd() {
declare APP="$1" PROC_TYPE="$2" START_CMD="$3" DOKKU_HEROKUISH="$4" PORT="$5" declare APP="$1" PROC_TYPE="$2" START_CMD="$3" DOKKU_HEROKUISH="$4" PORT="$5"
local DOKKU_DOCKERFILE_START_CMD DOKKU_PROCFILE_START_CMD START_CMD local DOKKU_DOCKERFILE_START_CMD DOKKU_PROCFILE_START_CMD START_CMD
if [[ "$DOKKU_HEROKUISH" != "false" ]]; then if [[ "$DOKKU_HEROKUISH" != "false" ]] && [[ -n "$START_CMD" ]]; then
echo "$START_CMD" echo "$START_CMD"
return return
fi fi
+1 -1
View File
@@ -7,7 +7,7 @@ cron: python3 worker.py
web: python3 web.py # testing inline comment web: python3 web.py # testing inline comment
worker: python3 worker.py worker: python3 worker.py
custom: echo -n custom: echo -n
release: touch /app/release.test release: python3 release.py
# Old version with separate processes (use this if you have issues with the threaded version) # Old version with separate processes (use this if you have issues with the threaded version)
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env python
import os
def main():
print("SECRET_KEY: {0}".format(os.getenv('SECRET_KEY')))
if __name__ == '__main__':
main()
+1
View File
@@ -0,0 +1 @@
flask
+26
View File
@@ -1,6 +1,12 @@
#!/usr/bin/env bats #!/usr/bin/env bats
load test_helper load test_helper
setup_file() {
add-apt-repository --yes ppa:cncf-buildpacks/pack-cli
apt-get update
apt-get --yes install pack-cli
}
setup() { setup() {
global_setup global_setup
create_app create_app
@@ -103,3 +109,23 @@ teardown() {
echo "status: $status" echo "status: $status"
assert_success assert_success
} }
@test "(app-json) app.json cnb release" {
run /bin/bash -c "dokku config:set --no-restart $TEST_APP DOKKU_CNB_EXPERIMENTAL=1 SECRET_KEY=fjdkslafjdk"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@dokku.me:$TEST_APP add_requirements_txt
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "Executing release task from Procfile"
assert_output_contains "SECRET_KEY: fjdkslafjdk"
run /bin/bash -c "curl $(dokku url $TEST_APP)/env"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains '"SECRET_KEY": "fjdkslafjdk"'
}