From ef95d2bcd4bac604441e55b56e0fde25da23653d Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 15 Jan 2017 18:02:52 -0500 Subject: [PATCH 01/51] [config] test some config oddities --- tests.mk | 10 +++++++ tests/unit/30_config-oddities.bats | 47 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/unit/30_config-oddities.bats diff --git a/tests.mk b/tests.mk index 08a9d65ca..59bf8d68c 100644 --- a/tests.mk +++ b/tests.mk @@ -91,6 +91,16 @@ else @$(QUIET) ./tests/ci/unit_test_runner.sh $$UNIT_TEST_BATCH endif +unit-tests-config: + @echo running config unit tests... + @$(QUIET) bats tests/unit/20_config.bats + +unit-tests-config-oddities: + @echo running config oddities unit tests... + @$(QUIET) bats tests/unit/30_config-oddities.bats + + + deploy-test-go-fail-predeploy: @echo deploying go-fail-predeploy app... cd tests && ./test_deploy ./apps/go-fail-predeploy dokku.me '' true diff --git a/tests/unit/30_config-oddities.bats b/tests/unit/30_config-oddities.bats new file mode 100644 index 000000000..681dbe91f --- /dev/null +++ b/tests/unit/30_config-oddities.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup + [[ -f ${DOKKU_ROOT}/ENV ]] && mv -f ${DOKKU_ROOT}/ENV ${DOKKU_ROOT}/ENV.bak + sudo -H -u dokku /bin/bash -c "echo 'export global_test=true' > ${DOKKU_ROOT}/ENV" + create_app +} + +teardown() { + destroy_app + ls -la ${DOKKU_ROOT} + if [[ -f ${DOKKU_ROOT}/ENV.bak ]];then + mv -f ${DOKKU_ROOT}/ENV.bak ${DOKKU_ROOT}/ENV + fi + global_teardown +} + +@test "(config-oddities) set-local/get with multiple spaces and $" { + run dokku config:set --global double_quotes="hello world$" single_quotes='hello$ world' + echo "output: "$output + echo "status: "$status + run dokku config:get --global double_quotes + echo "output: "$output + echo "status: "$status + assert_output 'hello world$' + run dokku config:get --global single_quotes + echo "output: "$output + echo "status: "$status + assert_output 'hello$ world' + assert_success +} + +@test "(config-oddities) set-local/get with multiple lines" { + multiline='line one + line two' + run dokku config:set --global double_quotes="$multiline" + echo "output: "$output + echo "status: "$status + run dokku config:get --global double_quotes + echo "output: "$output + echo "status: "$status + assert_output "$multiline" + assert_success +} \ No newline at end of file From e57e1408b26938c9d4569777053ed7b55c159420 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 16 Jan 2017 23:07:20 -0500 Subject: [PATCH 02/51] [common] add NewTokenizedShellCmd to ShellCmd --- plugins/common/common.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index 31060a760..e124c4a68 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -22,11 +22,16 @@ type ShellCmd struct { // NewShellCmd returns a new ShellCmd struct func NewShellCmd(command string) *ShellCmd { items := strings.Split(command, " ") - cmd := items[0] - args := items[1:] + return NewTokenizedShellCmd(items...) +} + +// NewTokenizedShellCmd creates a new ShellCmd +func NewTokenizedShellCmd(command ...string) *ShellCmd { + cmd := command[0] + args := command[1:] return &ShellCmd{ Command: exec.Command(cmd, args...), - CommandString: command, + CommandString: strings.Join(command, " "), Args: args, ShowOutput: true, } From 5e3184b61d6dda8f1535551e74f7cd4943b0b639 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 16 Jan 2017 23:08:02 -0500 Subject: [PATCH 03/51] [common] helper for go trigger plugn events --- plugins/common/common.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/common/common.go b/plugins/common/common.go index e124c4a68..7d0f1106e 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -144,3 +144,9 @@ func VerifyImage(image string) bool { } return false } + +//PlugnTrigger fire the given plugn trigger with the given args +func PlugnTrigger(triggerName string, args ...string) (string, error) { + res, err := sh.Command("plugn", "trigger", triggerName, args).Output() + return string(res), err +} From 294d4b1f2936981925f0852e2a9b7570aabee23c Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 16 Jan 2017 23:13:32 -0500 Subject: [PATCH 04/51] [config] first cut of config rewritten in golang still to do: * Rework disk format to be okay with quotes * Fix Dockerfile deployments * Dedup code in the subcommands * Add --encode * Add deploytests for the new stuff * Restore some log messages (should I make some helpers in common?) * Restore some functions that may be used by 3d-party plugins --- plugins/config/.gitignore | 2 + plugins/config/Makefile | 32 ++ plugins/config/commands | 33 -- plugins/config/functions | 243 +------------- plugins/config/src/commands/commands.go | 66 ++++ plugins/config/src/configenv/configenv.go | 139 ++++++++ .../config/src/configenv/configenv_test.go | 42 +++ plugins/config/src/configenv/parsing.go | 119 +++++++ plugins/config/src/glide.lock | 6 + plugins/config/src/glide.yaml | 3 + .../config/src/subcommands/export/export.go | 40 +++ plugins/config/src/subcommands/get/get.go | 39 +++ plugins/config/src/subcommands/set/set.go | 61 ++++ plugins/config/src/subcommands/unset/unset.go | 54 ++++ .../github.com/ryanuber/columnize/.travis.yml | 3 + .../github.com/ryanuber/columnize/LICENSE | 20 ++ .../github.com/ryanuber/columnize/README.md | 69 ++++ .../ryanuber/columnize/columnize.go | 178 ++++++++++ .../ryanuber/columnize/columnize_test.go | 306 ++++++++++++++++++ plugins/config/subcommands/default | 11 - plugins/config/subcommands/get | 11 - plugins/config/subcommands/set | 11 - plugins/config/subcommands/unset | 11 - 23 files changed, 1187 insertions(+), 312 deletions(-) create mode 100644 plugins/config/.gitignore create mode 100644 plugins/config/Makefile delete mode 100755 plugins/config/commands create mode 100644 plugins/config/src/commands/commands.go create mode 100644 plugins/config/src/configenv/configenv.go create mode 100644 plugins/config/src/configenv/configenv_test.go create mode 100644 plugins/config/src/configenv/parsing.go create mode 100644 plugins/config/src/glide.lock create mode 100644 plugins/config/src/glide.yaml create mode 100644 plugins/config/src/subcommands/export/export.go create mode 100644 plugins/config/src/subcommands/get/get.go create mode 100644 plugins/config/src/subcommands/set/set.go create mode 100644 plugins/config/src/subcommands/unset/unset.go create mode 100644 plugins/config/src/vendor/github.com/ryanuber/columnize/.travis.yml create mode 100644 plugins/config/src/vendor/github.com/ryanuber/columnize/LICENSE create mode 100644 plugins/config/src/vendor/github.com/ryanuber/columnize/README.md create mode 100644 plugins/config/src/vendor/github.com/ryanuber/columnize/columnize.go create mode 100644 plugins/config/src/vendor/github.com/ryanuber/columnize/columnize_test.go delete mode 100755 plugins/config/subcommands/default delete mode 100755 plugins/config/subcommands/get delete mode 100755 plugins/config/subcommands/set delete mode 100755 plugins/config/subcommands/unset diff --git a/plugins/config/.gitignore b/plugins/config/.gitignore new file mode 100644 index 000000000..0f6770b1e --- /dev/null +++ b/plugins/config/.gitignore @@ -0,0 +1,2 @@ +/commands +/subcommands/* diff --git a/plugins/config/Makefile b/plugins/config/Makefile new file mode 100644 index 000000000..8718ca0bc --- /dev/null +++ b/plugins/config/Makefile @@ -0,0 +1,32 @@ +.PHONY: build-in-docker build clean + +GO_REPO_ROOT ?= /go/src/github.com/dokku/dokku +BUILD_IMAGE ?= golang:1.7.1 + +build-in-docker: clean + docker run --rm \ + -v $$PWD/../..:$(GO_REPO_ROOT) \ + -w $(GO_REPO_ROOT)/plugins/config \ + $(BUILD_IMAGE) \ + bash -c "make build" || exit $$? + +build: commands subcommands +subcommands: subcommands/export subcommands/set subcommands/get subcommands/unset + +commands: **/**/commands.go + go build -a -o commands src/commands/commands.go + +subcommands/export: **/**/**/export.go + go build -a -o subcommands/export src/subcommands/export/export.go + +subcommands/set: **/**/**/set.go + go build -a -o subcommands/set src/subcommands/set/set.go + +subcommands/get: **/**/**/get.go + go build -a -o subcommands/get src/subcommands/get/get.go + +subcommands/unset: **/**/**/unset.go + go build -a -o subcommands/unset src/subcommands/unset/unset.go + +clean: + rm -rf commands subcommands diff --git a/plugins/config/commands b/plugins/config/commands deleted file mode 100755 index 305cb27ac..000000000 --- a/plugins/config/commands +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -[[ " help config:help " == *" $1 "* ]] || exit "$DOKKU_NOT_IMPLEMENTED_EXIT" -set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x - -case "$1" in - help | config:help) - help_content_func () { - declare desc="return config plugin help content" - cat<|--global), Display all global or app-specific config vars - config:get (|--global) KEY, Display a global or app-specific config value - config:set (|--global) KEY1=VALUE1 [KEY2=VALUE2 ...], Set one or more config vars - config:unset (|--global) KEY1 [KEY2 ...], Unset one or more config vars -help_content - } - - if [[ $1 = "config:help" ]] ; then - echo -e 'Usage: dokku config[:COMMAND] (|--global)' - echo '' - echo 'Manage global or app-specific config vars.' - echo '' - echo 'Additional commands:' - help_content_func | sort | column -c2 -t -s, - else - help_content_func - fi - ;; - - *) - exit "$DOKKU_NOT_IMPLEMENTED_EXIT" - ;; - -esac diff --git a/plugins/config/functions b/plugins/config/functions index 908f6a7af..582d49018 100755 --- a/plugins/config/functions +++ b/plugins/config/functions @@ -6,251 +6,24 @@ config_export() { declare desc="returns export command for config variable of specified type (app/global)" local CONFIG_TYPE="$1" local APP="$2" - local ENV_FILE="$DOKKU_ROOT/$APP/ENV" - - [[ $CONFIG_TYPE == "global" ]] && local ENV_FILE="$DOKKU_ROOT/ENV" - [[ ! -f $ENV_FILE ]] && return 0 - [[ ! -s $ENV_FILE ]] && return 0 - - local VARS=$(grep -Eo "export ([a-zA-Z_][a-zA-Z0-9_]*=.*)" "$ENV_FILE" | cut -d" " -f2-) - echo "$VARS" | awk '{print "export " $0}' + if [[ $CONFIG_TYPE == "global" ]]; then + APP="--global" + fi + dokku config:export "$APP" return 0 } -get_sanitized_config_args() { - local desc="return sanitized arguments for config plugin" - local SANITIZED_ARGS=("$@") - - SANITIZED_ARGS=("${SANITIZED_ARGS[@]//--global/}") - SANITIZED_ARGS=("${SANITIZED_ARGS[@]//--no-restart/}") - SANITIZED_ARGS=("${SANITIZED_ARGS[@]//--export/}") - SANITIZED_ARGS=("${SANITIZED_ARGS[@]//--shell/}") - - echo "${SANITIZED_ARGS[@]}" -} - -config_parse_args() { - declare desc="parse config plugin args" - unset APP ENV_FILE DOKKU_CONFIG_TYPE DOKKU_CONFIG_RESTART DOKKU_CONFIG_EXPORT - - for var; do - if [[ "$var" == "--global" ]]; then - ENV_FILE="$DOKKU_ROOT/ENV" - DOKKU_CONFIG_TYPE="global" - DOKKU_CONFIG_RESTART=false - fi - if [[ "$var" == "--no-restart" ]]; then - DOKKU_CONFIG_RESTART=false - fi - if [[ "$var" == "--export" ]]; then - DOKKU_CONFIG_EXPORT=true - fi - if [[ "$var" == "--shell" ]]; then - DOKKU_CONFIG_SHELL=true - fi - done - - local SANITIZED_ARGS=($(get_sanitized_config_args "$@")) - set -- "${SANITIZED_ARGS[@]}" - - DOKKU_CONFIG_TYPE=${DOKKU_CONFIG_TYPE:="app"} - DOKKU_CONFIG_RESTART=${DOKKU_CONFIG_RESTART:=true} - - if [[ "$DOKKU_CONFIG_TYPE" == "app" ]]; then - if [[ -z $2 ]]; then - dokku_log_fail "Please specify an app to run the command on" - else - verify_app_name "$2" - APP=${APP:="$2"} - ENV_FILE=${ENV_FILE:="$DOKKU_ROOT/$APP/ENV"} - fi - fi - export APP ENV_FILE DOKKU_CONFIG_TYPE DOKKU_CONFIG_RESTART DOKKU_CONFIG_EXPORT -} - -config_create () { - declare desc="create config env file" - local ENV_FILE=$1 - [[ -f $ENV_FILE ]] || touch "$ENV_FILE" -} - -config_styled_hash () { - declare desc="internal config hash" - local vars="$1" - local prefix="$2" - local longest="" - - while read -r word; do - local KEY=$(echo "$word" | cut -d"=" -f1) - if [[ ${#KEY} -gt ${#longest} ]]; then - local longest=$KEY - fi - done <<< "$vars" - - while read -r word; do - local KEY=$(echo "$word" | cut -d"=" -f1) - local VALUE=$(echo "$word" | cut -d"=" -f2- | sed -e "s/^'//" -e "s/'$//" -e "s/\$$//g") - - local num_zeros=$((${#longest} - ${#KEY})) - local zeros=" " - while [[ $num_zeros -gt 0 ]]; do - local zeros="$zeros " - local num_zeros=$((num_zeros - 1)) - done - echo "$prefix$KEY:$zeros$VALUE" - done <<< "$vars" -} - -config_write() { - declare desc="writes config vars out to appropriate file path" - local ENV_TEMP="$1" - local ENV_FILE_TEMP="${ENV_FILE}.tmp" - - echo "$ENV_TEMP" | sed '/^$/d' | sort > "$ENV_FILE_TEMP" - if ! cmp -s "$ENV_FILE" "$ENV_FILE_TEMP"; then - cp -f "$ENV_FILE_TEMP" "$ENV_FILE" - chmod 600 "$ENV_FILE" - fi - rm -f "$ENV_FILE_TEMP" -} - -config_all() { - declare desc="print or export config vars" - [[ "$1" == "config" ]] || set -- "config" "$@" - config_parse_args "$@" - local SANITIZED_ARGS=($(get_sanitized_config_args "$@")) - set -- "${SANITIZED_ARGS[@]}" - - config_create "$ENV_FILE" - [[ $DOKKU_CONFIG_EXPORT ]] && config_export "$DOKKU_CONFIG_TYPE" "$APP" && return 0 - - [[ $APP ]] && local DOKKU_CONFIG_TYPE=$APP - [[ ! -s $ENV_FILE ]] && dokku_log_fail "no config vars for $DOKKU_CONFIG_TYPE" - - local VARS=$(grep -Eo "export ([a-zA-Z_][a-zA-Z0-9_]*=.*)" "$ENV_FILE" | cut -d" " -f2-) - - [[ $DOKKU_CONFIG_SHELL ]] && echo "$VARS" && return 0 - - dokku_log_info2_quiet "$DOKKU_CONFIG_TYPE config vars" - config_styled_hash "$VARS" -} - config_get() { declare desc="get value of given config var" - [[ "$1" == "config:get" ]] || set -- "config:get" "$@" - config_parse_args "$@" - local SANITIZED_ARGS=($(get_sanitized_config_args "$@")) - set -- "${SANITIZED_ARGS[@]}" - - if [[ -z $2 ]]; then - echo "Usage: dokku config:get APP KEY" - echo "Must specify KEY." - exit 1 - fi - - config_create "$ENV_FILE" - if [[ ! -s $ENV_FILE ]]; then - return 1 - fi - - local KEY="${*: -1}" - - grep -Eo "export ([a-zA-Z_][a-zA-Z0-9_]*=.*)" "$ENV_FILE" | grep "^export $KEY=" | cut -d"=" -f2- | sed -e "s/^'//" -e "s/'$//" + dokku config:get "$@" } config_set() { declare desc="set value of given config var" - source "$PLUGIN_AVAILABLE_PATH/ps/functions" - - [[ "$1" == "config:set" ]] || set -- "config:set" "$@" - config_parse_args "$@" - - if [[ -z "${*:3}" ]]; then - echo "Usage: dokku config:set APP KEY1=VALUE1 [KEY2=VALUE2 ...]" - echo "Whitespaces need to be escaped, i.e. KEY=\"VAL\ WITH\ SPACES\"" - echo "Must specify KEY and VALUE to set." - exit 1 - fi - - config_create "$ENV_FILE" - local ENV_ADD="" - local ENV_TEMP=$(cat "${ENV_FILE}") - local RESTART_APP=false - shift 2 - - for var; do - if [[ $var == "--global" ]] || [[ $var == "--no-restart" ]] || ([[ $APP ]] && [[ $var == "$APP" ]]); then - shift 1 - elif [[ $var != *"="* ]]; then - echo "Usage: dokku config:set APP KEY1=VALUE1 [KEY2=VALUE2 ...]" - echo "Whitespaces need to be escaped, i.e. KEY=\"VAL\ WITH\ SPACES\"" - echo "Must specify KEY and VALUE to set." - exit 1 - fi - done - - for var; do - local KEY=$(echo ${var} | cut -d"=" -f1) - local VALUE=$(echo ${var} | cut -d"=" -f2-) - - if [[ $KEY =~ [a-zA-Z_][a-zA-Z0-9_]* ]]; then - local RESTART_APP=true - local ENV_TEMP=$(echo "${ENV_TEMP}" | sed "/^export $KEY=/ d") - local ENV_TEMP="${ENV_TEMP} -export $KEY='$VALUE'" - local ENV_ADD=$(echo -e "${ENV_ADD}" | sed "/^$KEY=/ d") - local ENV_ADD="${ENV_ADD}$ -${var}" - fi - done - local ENV_ADD=$(echo "$ENV_ADD" | tail -n +2) #remove first empty line - - if [[ $RESTART_APP ]]; then - dokku_log_info1 "Setting config vars" - config_styled_hash "$ENV_ADD" " " - - config_write "$ENV_TEMP" - plugn trigger post-config-update "$APP" "set" "$@" - fi - - [[ $APP ]] && local DOKKU_APP_RESTORE=$(config_get "$APP" DOKKU_APP_RESTORE || true) - if [[ "$DOKKU_CONFIG_RESTART" == "true" ]] && [[ "$DOKKU_APP_RESTORE" != 0 ]]; then - dokku_log_info1 "Restarting app $APP" - ps_restart "$APP" - fi + dokku config:set "$@" } config_unset() { - declare desc="unset given config var" - source "$PLUGIN_AVAILABLE_PATH/ps/functions" - - [[ "$1" == "config:unset" ]] || set -- "config:unset" "$@" - config_parse_args "$@" - local SANITIZED_ARGS=($(get_sanitized_config_args "$@")) - set -- "${SANITIZED_ARGS[@]}" - - if [[ -z $2 ]]; then - echo "Usage: dokku config:unset APP KEY1 [KEY2 ...]" - echo "Must specify KEY to unset." - exit 1 - fi - - config_create "$ENV_FILE" - local ENV_TEMP=$(cat "${ENV_FILE}") - local VARS="${*:2}" - - for var in $VARS; do - dokku_log_info1 "Unsetting $var" - local ENV_TEMP=$(echo "${ENV_TEMP}" | sed "/^export $var=/ d") - - config_write "$ENV_TEMP" - done - - plugn trigger post-config-update "$APP" "unset" "$@" - - [[ $APP ]] && local DOKKU_APP_RESTORE=$(config_get "$APP" DOKKU_APP_RESTORE || true) - if [[ "$DOKKU_CONFIG_RESTART" == "true" ]] && [[ "$DOKKU_APP_RESTORE" != 0 ]]; then - dokku_log_info1 "Restarting app $APP" - ps_restart "$APP" - fi + declare desc="set value of given config var" + dokku config:unset "$@" } diff --git a/plugins/config/src/commands/commands.go b/plugins/config/src/commands/commands.go new file mode 100644 index 000000000..d0a04dbf3 --- /dev/null +++ b/plugins/config/src/commands/commands.go @@ -0,0 +1,66 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + common "github.com/dokku/dokku/plugins/common" + configenv "github.com/dokku/dokku/plugins/config/src/configenv" + columnize "github.com/ryanuber/columnize" +) + +const ( + helpHeader = `Usage: dokku config [|--global] + +Display all global or app-specific config vars + +Additional commands:` + + helpContent = ` + config:get (|--global) KEY, Display a global or app-specific config value + config:set (|--global) [--no-restart] KEY1=VALUE1 [KEY2=VALUE2 ...], Set one or more config vars + config:unset (|--global) KEY1 [KEY2 ...], Unset one or more config vars + config:export (|--global) [--envfile], Export a global or app environment +` +) + +func main() { + flag.Usage = usage + flag.Parse() + + cmd := flag.Arg(0) + switch cmd { + case "config": + target := flag.Arg(1) + env, err := configenv.NewFromTarget(target) + if err != nil { + common.LogFail(err.Error()) + } else { + fmt.Println(env.String()) + } + case "config:help": + usage() + case "help": + fmt.Print(helpContent) + default: + dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT")) + if err != nil { + fmt.Println("failed to retrieve DOKKU_NOT_IMPLEMENTED_EXIT environment variable") + dokkuNotImplementExitCode = 10 + } + os.Exit(dokkuNotImplementExitCode) + } +} + +func usage() { + config := columnize.DefaultConfig() + config.Delim = "," + config.Prefix = "\t" + config.Empty = "" + content := strings.Split(helpContent, "\n")[1:] + fmt.Println(helpHeader) + fmt.Println(columnize.Format(content, config)) +} diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go new file mode 100644 index 000000000..d449c7bbe --- /dev/null +++ b/plugins/config/src/configenv/configenv.go @@ -0,0 +1,139 @@ +package configenv + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + common "github.com/dokku/dokku/plugins/common" +) + +//Env is a representation for global or app environment +type Env struct { + name string + env map[string]string + filename string +} + +func (e *Env) String() string { + return e.EnvfileString() +} + +//EnvfileString returns the contents of this Env in ENVFILE format +func (e *Env) EnvfileString() string { + return e.stringWithEntryPrefix("") +} + +//ExportfileString returns the contents of this Env as bash exports +func (e *Env) ExportfileString() string { + return e.stringWithEntryPrefix("export ") +} + +func (e *Env) stringWithEntryPrefix(prefix string) string { + keys := e.Keys() + sort.Strings(keys) + entries := make([]string, len(keys)) + for i, k := range keys { + entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, escape(e.env[k])) + } + return strings.Join(entries, "\n") +} + +func escape(value string) string { //TODO: improve this + return strings.Replace(value, "'", "\\'", -1) +} + +//NewFromTarget creates an env from the given target. Tareget is either "--global" or an app name +func NewFromTarget(target string) (*Env, error) { + if target == "--global" { + return parseEnv("global", getGlobalFile()) + } + appfile, err := getAppFile(target) + if err != nil { + return nil, err + } + return parseEnv(target, appfile) +} + +//NewFromString creates an env from the given ENVFILE contents representation +func NewFromString(rep string) (*Env, error) { + return parseEnvFromReader("", "", strings.NewReader(rep)) +} + +//Set an environment variable +func (e *Env) Set(key string, value string) { + e.env[key] = value +} + +//Unset an environment variable +func (e *Env) Unset(key string) { + delete(e.env, key) +} + +//Keys gets the keys in this environment +func (e *Env) Keys() []string { + keys := make([]string, 0, len(e.env)) + for k := range e.env { + keys = append(keys, k) + } + return keys +} + +//Get an environment variable +func (e *Env) Get(key string) (string, bool) { + v, ok := e.env[key] + return v, ok +} + +//GetDefault an environment variable or a default if it doesnt exist +func (e *Env) GetDefault(key string, defaultValue string) string { + v, ok := e.env[key] + if !ok { + return defaultValue + } + return v +} + +//GetBoolDefault gets the bool value of the given key with the given default +//right now that is evaluated as `value != "0"` +func (e *Env) GetBoolDefault(key string, defaultValue bool) bool { + v, ok := e.Get(key) + if !ok { + return defaultValue + } + return v != "0" +} + +//Map return the Env as a map +func (e *Env) Map() map[string]string { + return e.env +} + +//Write an Env back to the file it was read from as an exportfile +func (e *Env) Write() error { + if e.filename == "" { + return errors.New("this Env was created unbound to a file") + } + file, err := os.Create(e.filename) + defer file.Close() + if err != nil { + return err + } + _, err = file.WriteString(e.ExportfileString()) + return err +} + +func getAppFile(appName string) (string, error) { + err := common.VerifyAppName(appName) + if err != nil { + return "", err + } + return filepath.Join(common.MustGetEnv("DOKKU_ROOT"), appName, "ENV"), nil +} + +func getGlobalFile() string { + return filepath.Join(common.MustGetEnv("DOKKU_ROOT"), "ENV") +} diff --git a/plugins/config/src/configenv/configenv_test.go b/plugins/config/src/configenv/configenv_test.go new file mode 100644 index 000000000..d0b3cbf02 --- /dev/null +++ b/plugins/config/src/configenv/configenv_test.go @@ -0,0 +1,42 @@ +package configenv + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestBasic(t *testing.T) { + RegisterTestingT(t) + env, err := NewFromString("HI=ho") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("HI", "ho"))) + Expect(env.String()).To(Equal("HI='ho'")) + + env, err = NewFromString("HI=h\\o\\'") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("HI", "h\\o'"))) + Expect(env.String()).To(Equal("HI='h\\o\\''")) + + env, err = NewFromString("HI=ho\nFOO=bar\n\n") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("HI", "ho", "FOO", "bar"))) + Expect(env.String()).To(Equal("FOO='bar'\nHI='ho'")) + + env, err = NewFromString("HI='ho\n'\n\nFOO='bar\\'\nbaz'") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("HI", "ho\n", "FOO", "bar'\nbaz"))) + Expect(env.String()).To(Equal("FOO='bar\\'\nbaz'\nHI='ho\n'")) +} + +func pairs(vars ...string) map[string]string { + res := map[string]string{} + var i = 0 + for i < len(vars)-1 { + key := vars[i] + value := vars[i+1] + res[key] = value + i += 2 + } + return res +} diff --git a/plugins/config/src/configenv/parsing.go b/plugins/config/src/configenv/parsing.go new file mode 100644 index 000000000..bf539156b --- /dev/null +++ b/plugins/config/src/configenv/parsing.go @@ -0,0 +1,119 @@ +package configenv + +import ( + "bufio" + "bytes" + "errors" + "io" + "os" +) + +func parseEnv(name string, filename string) (*Env, error) { + var file *os.File + var err error + if _, err := os.Stat(filename); os.IsNotExist(err) { + file, err = os.Create(filename) + } else { + file, err = os.Open(filename) + } + if err != nil { + return nil, err + } + defer file.Close() + return parseEnvFromReader(name, filename, file) +} + +func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, error) { + //The file format looks like KEY='VALUE'. Eait pair is terminated with a newline. + //All characters are valid for VALUE without escaping + //but one: the single quote and backslash. To represent a single quote emit \'. To represent + //a literal backslash emit \\. In any other case, the backslash is interpreted literally. + //KEY='value + //one' + //KEY='val\\ue\\'' etc.. + + //TODO: We might want to rework this to use shell-style quoting for single quotes + buffered := bufio.NewReader(reader) + const ( + StateKey = iota + StateValue = iota + ) + env := &Env{name, map[string]string{}, filename} + var buffer bytes.Buffer + var state = StateKey + var quoted = false + var escaped = false + var key = "" + for char, _, err := buffered.ReadRune(); err == nil; char, _, err = buffered.ReadRune() { + switch state { + case StateKey: + switch char { + case ' ': + if buffer.String() == "export" { + buffer.Truncate(0) //so we can read exportfiles too + } else { + return nil, errors.New("Env keys cannot have spaces") + } + case '=': + key = buffer.String() + buffer.Truncate(0) + state = StateValue + case '\n': + if buffer.Len() > 0 { + return nil, errors.New("Invalid newline after: " + buffer.String()) + } + default: + buffer.WriteRune(char) + } + case StateValue: + switch char { + case '\'': + if escaped { + buffer.WriteRune('\'') + escaped = false + } else { + quoted = !quoted + } + case '\n': + if escaped || quoted { + buffer.WriteRune(char) + } else { + state = StateKey + env.Set(key, buffer.String()) + buffer.Truncate(0) + key = "" + escaped = false + quoted = false + } + case '\\': + if escaped { + buffer.WriteRune(char) + escaped = false + } else { + escaped = true + } + default: + if escaped { + buffer.WriteRune('\\') + escaped = false + } + buffer.WriteRune(char) + } + } + } + if escaped { + return nil, errors.New("Unterminated escape") + } + if quoted { + return nil, errors.New("Unterminated quote") + } + + if state == StateValue { + env.Set(key, buffer.String()) + } else { + if buffer.Len() > 0 { + return nil, errors.New("Invalid trailing content: " + buffer.String()) + } + } + return env, nil +} diff --git a/plugins/config/src/glide.lock b/plugins/config/src/glide.lock new file mode 100644 index 000000000..fe4a98f07 --- /dev/null +++ b/plugins/config/src/glide.lock @@ -0,0 +1,6 @@ +hash: 1ddab5de41d1514c2722bd7e24758ad4b60bf6956eb5b9b925fa071a1427f149 +updated: 2017-01-03T17:16:50.97156327-08:00 +imports: +- name: github.com/ryanuber/columnize + version: 0fbbb3f0e3fbdc5bae7c6cd5f6c1887ebfb76360 +testImports: [] diff --git a/plugins/config/src/glide.yaml b/plugins/config/src/glide.yaml new file mode 100644 index 000000000..3f7a96657 --- /dev/null +++ b/plugins/config/src/glide.yaml @@ -0,0 +1,3 @@ +package: . +import: +- package: github.com/ryanuber/columnize diff --git a/plugins/config/src/subcommands/export/export.go b/plugins/config/src/subcommands/export/export.go new file mode 100644 index 000000000..52feae877 --- /dev/null +++ b/plugins/config/src/subcommands/export/export.go @@ -0,0 +1,40 @@ +package main + +import ( + "flag" + "fmt" + "os" + + common "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config/src/configenv" +) + +// print the environment to stdout +func main() { + args := flag.NewFlagSet("config:export", flag.ExitOnError) + global := args.Bool("global", false, "--global: use the global environment") + envfile := args.Bool("envfile", false, "--envfile: export as envfile rather than bash exports") + args.Parse(os.Args[2:]) + + appName := args.Arg(0) + if appName == "" && !*global { + common.LogFail("Please specify an app or --global") + } + + if *global { + if appName != "" { + common.LogFail("Trailing argument: " + appName) + } + appName = "--global" + } + + env, err := configenv.NewFromTarget(appName) + if err != nil { + common.LogFail(err.Error()) + } + if *envfile { + fmt.Println(env.EnvfileString()) + } else { + fmt.Println(env.ExportfileString()) + } +} diff --git a/plugins/config/src/subcommands/get/get.go b/plugins/config/src/subcommands/get/get.go new file mode 100644 index 000000000..3c57eea71 --- /dev/null +++ b/plugins/config/src/subcommands/get/get.go @@ -0,0 +1,39 @@ +package main + +import ( + "flag" + "fmt" + "os" + + common "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config/src/configenv" +) + +// get the given entries from the specified environment +func main() { + args := flag.NewFlagSet("config:get", flag.ExitOnError) + global := args.Bool("global", false, "--global: use the global environment") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + nextArg := 0 + if appName == "" && !*global { + common.LogFail("Please specify an app or --global") + } + + if *global { + appName = "--global" + } else { + nextArg = 1 + } + + env, err := configenv.NewFromTarget(appName) + if err != nil { + common.LogFail(err.Error()) + } + + if args.NArg() > nextArg+1 { + common.LogFail(fmt.Sprintf("Unexpected argument(s): %v", args.Args()[nextArg+1:])) + } + key := args.Arg(nextArg) + fmt.Printf("%s", env.GetDefault(key, "")) +} diff --git a/plugins/config/src/subcommands/set/set.go b/plugins/config/src/subcommands/set/set.go new file mode 100644 index 000000000..5e0b38dab --- /dev/null +++ b/plugins/config/src/subcommands/set/set.go @@ -0,0 +1,61 @@ +package main + +import ( + "flag" + + "strings" + + "os" + + common "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config/src/configenv" +) + +// set the given entries from the specified environment +func main() { + args := flag.NewFlagSet("config:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: use the global environment") + noRestart := args.Bool("no-restart", false, "--no-restart: no restart") + args.Parse(os.Args[2:]) + shouldRestart := !*global && !*noRestart + var nextArg = 0 + appName := args.Arg(0) + if appName == "" && !*global { + common.LogFail("Please specify an app or --global") + } + + if *global { + appName = "--global" + } else { + nextArg = 1 + } + + env, err := configenv.NewFromTarget(appName) + if err != nil { + common.LogFail(err.Error()) + } + + var changed = false + entries := args.Args()[nextArg:] + for _, e := range entries { + //log + parts := strings.SplitN(e, "=", 2) + if len(parts) == 1 { + common.LogFail("Invalid env pair: " + e) + } + env.Set(parts[0], parts[1]) + changed = true + } + + if changed { + //log + env.Write() + args := append([]string{appName, "set"}, entries...) + common.PlugnTrigger("post-config-update", args...) + } + + if shouldRestart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { + cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) + cmd.Execute() + } +} diff --git a/plugins/config/src/subcommands/unset/unset.go b/plugins/config/src/subcommands/unset/unset.go new file mode 100644 index 000000000..ed72cf44b --- /dev/null +++ b/plugins/config/src/subcommands/unset/unset.go @@ -0,0 +1,54 @@ +package main + +import ( + "flag" + "os" + + common "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config/src/configenv" +) + +//unset the given entries from the given environment +func main() { + args := flag.NewFlagSet("config:unset", flag.ExitOnError) + global := args.Bool("global", false, "--global: use the global environment") + noRestart := args.Bool("no-restart", false, "--no-restart: no restart") + args.Parse(os.Args[2:]) + shouldRestart := !*global && !*noRestart + var nextArg = 0 + appName := args.Arg(0) + if appName == "" && !*global { + common.LogFail("Please specify an app or --global") + } + + if *global { + appName = "--global" + } else { + nextArg = 1 + } + + env, err := configenv.NewFromTarget(appName) + if err != nil { + common.LogFail(err.Error()) + } + + var changed = false + keys := args.Args()[nextArg:] + for _, k := range keys { + //log + env.Unset(k) + changed = true + } + + if changed { + //log + env.Write() + args := append([]string{appName, "unset"}, keys...) + common.PlugnTrigger("post-config-update", args...) + } + + if shouldRestart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { + cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) + cmd.Execute() + } +} diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/.travis.yml b/plugins/config/src/vendor/github.com/ryanuber/columnize/.travis.yml new file mode 100644 index 000000000..1a0bbea6c --- /dev/null +++ b/plugins/config/src/vendor/github.com/ryanuber/columnize/.travis.yml @@ -0,0 +1,3 @@ +language: go +go: + - tip diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/LICENSE b/plugins/config/src/vendor/github.com/ryanuber/columnize/LICENSE new file mode 100644 index 000000000..b9c0e2b68 --- /dev/null +++ b/plugins/config/src/vendor/github.com/ryanuber/columnize/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2016 Ryan Uber + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/README.md b/plugins/config/src/vendor/github.com/ryanuber/columnize/README.md new file mode 100644 index 000000000..e47634fc6 --- /dev/null +++ b/plugins/config/src/vendor/github.com/ryanuber/columnize/README.md @@ -0,0 +1,69 @@ +Columnize +========= + +Easy column-formatted output for golang + +[![Build Status](https://travis-ci.org/ryanuber/columnize.svg)](https://travis-ci.org/ryanuber/columnize) +[![GoDoc](https://godoc.org/github.com/ryanuber/columnize?status.svg)](https://godoc.org/github.com/ryanuber/columnize) + +Columnize is a really small Go package that makes building CLI's a little bit +easier. In some CLI designs, you want to output a number similar items in a +human-readable way with nicely aligned columns. However, figuring out how wide +to make each column is a boring problem to solve and eats your valuable time. + +Here is an example: + +```go +package main + +import ( + "fmt" + "github.com/ryanuber/columnize" +) + +func main() { + output := []string{ + "Name | Gender | Age", + "Bob | Male | 38", + "Sally | Female | 26", + } + result := columnize.SimpleFormat(output) + fmt.Println(result) +} +``` + +As you can see, you just pass in a list of strings. And the result: + +``` +Name Gender Age +Bob Male 38 +Sally Female 26 +``` + +Columnize is tolerant of missing or empty fields, or even empty lines, so +passing in extra lines for spacing should show up as you would expect. + +Configuration +============= + +Columnize is configured using a `Config`, which can be obtained by calling the +`DefaultConfig()` method. You can then tweak the settings in the resulting +`Config`: + +``` +config := columnize.DefaultConfig() +config.Delim = "|" +config.Glue = " " +config.Prefix = "" +config.Empty = "" +``` + +* `Delim` is the string by which columns of **input** are delimited +* `Glue` is the string by which columns of **output** are delimited +* `Prefix` is a string by which each line of **output** is prefixed +* `Empty` is a string used to replace blank values found in output + +You can then pass the `Config` in using the `Format` method (signature below) to +have text formatted to your liking. + +See the [godoc](https://godoc.org/github.com/ryanuber/columnize) page for usage. diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize.go b/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize.go new file mode 100644 index 000000000..915716a10 --- /dev/null +++ b/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize.go @@ -0,0 +1,178 @@ +package columnize + +import ( + "bytes" + "fmt" + "strings" +) + +// Config can be used to tune certain parameters which affect the way +// in which Columnize will format output text. +type Config struct { + // The string by which the lines of input will be split. + Delim string + + // The string by which columns of output will be separated. + Glue string + + // The string by which columns of output will be prefixed. + Prefix string + + // A replacement string to replace empty fields + Empty string +} + +// DefaultConfig returns a *Config with default values. +func DefaultConfig() *Config { + return &Config{ + Delim: "|", + Glue: " ", + Prefix: "", + Empty: "", + } +} + +// MergeConfig merges two config objects together and returns the resulting +// configuration. Values from the right take precedence over the left side. +func MergeConfig(a, b *Config) *Config { + var result Config = *a + + // Return quickly if either side was nil + if a == nil || b == nil { + return &result + } + + if b.Delim != "" { + result.Delim = b.Delim + } + if b.Glue != "" { + result.Glue = b.Glue + } + if b.Prefix != "" { + result.Prefix = b.Prefix + } + if b.Empty != "" { + result.Empty = b.Empty + } + + return &result +} + +// stringFormat, given a set of column widths and the number of columns in +// the current line, returns a sprintf-style format string which can be used +// to print output aligned properly with other lines using the same widths set. +func stringFormat(c *Config, widths []int, columns int) string { + // Create the buffer with an estimate of the length + buf := bytes.NewBuffer(make([]byte, 0, (6+len(c.Glue))*columns)) + + // Start with the prefix, if any was given. The buffer will not return an + // error so it does not need to be handled + buf.WriteString(c.Prefix) + + // Create the format string from the discovered widths + for i := 0; i < columns && i < len(widths); i++ { + if i == columns-1 { + buf.WriteString("%s\n") + } else { + fmt.Fprintf(buf, "%%-%ds%s", widths[i], c.Glue) + } + } + return buf.String() +} + +// elementsFromLine returns a list of elements, each representing a single +// item which will belong to a column of output. +func elementsFromLine(config *Config, line string) []interface{} { + seperated := strings.Split(line, config.Delim) + elements := make([]interface{}, len(seperated)) + for i, field := range seperated { + value := strings.TrimSpace(field) + + // Apply the empty value, if configured. + if value == "" && config.Empty != "" { + value = config.Empty + } + elements[i] = value + } + return elements +} + +// runeLen calculates the number of visible "characters" in a string +func runeLen(s string) int { + l := 0 + for _ = range s { + l++ + } + return l +} + +// widthsFromLines examines a list of strings and determines how wide each +// column should be considering all of the elements that need to be printed +// within it. +func widthsFromLines(config *Config, lines []string) []int { + widths := make([]int, 0, 8) + + for _, line := range lines { + elems := elementsFromLine(config, line) + for i := 0; i < len(elems); i++ { + l := runeLen(elems[i].(string)) + if len(widths) <= i { + widths = append(widths, l) + } else if widths[i] < l { + widths[i] = l + } + } + } + return widths +} + +// Format is the public-facing interface that takes a list of strings and +// returns nicely aligned column-formatted text. +func Format(lines []string, config *Config) string { + conf := MergeConfig(DefaultConfig(), config) + widths := widthsFromLines(conf, lines) + + // Estimate the buffer size + glueSize := len(conf.Glue) + var size int + for _, w := range widths { + size += w + glueSize + } + size *= len(lines) + + // Create the buffer + buf := bytes.NewBuffer(make([]byte, 0, size)) + + // Create a cache for the string formats + fmtCache := make(map[int]string, 16) + + // Create the formatted output using the format string + for _, line := range lines { + elems := elementsFromLine(conf, line) + + // Get the string format using cache + numElems := len(elems) + stringfmt, ok := fmtCache[numElems] + if !ok { + stringfmt = stringFormat(conf, widths, numElems) + fmtCache[numElems] = stringfmt + } + + fmt.Fprintf(buf, stringfmt, elems...) + } + + // Get the string result + result := buf.String() + + // Remove trailing newline without removing leading/trailing space + if n := len(result); n > 0 && result[n-1] == '\n' { + result = result[:n-1] + } + + return result +} + +// SimpleFormat is a convenience function to format text with the defaults. +func SimpleFormat(lines []string) string { + return Format(lines, nil) +} diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize_test.go b/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize_test.go new file mode 100644 index 000000000..89dabaa38 --- /dev/null +++ b/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize_test.go @@ -0,0 +1,306 @@ +package columnize + +import ( + "fmt" + "testing" + + crand "crypto/rand" +) + +func TestListOfStringsInput(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "x | y | z", + } + + config := DefaultConfig() + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestEmptyLinesOutput(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "", + "x | y | z", + } + + config := DefaultConfig() + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestLeadingSpacePreserved(t *testing.T) { + input := []string{ + "| Column B | Column C", + "x | y | z", + } + + config := DefaultConfig() + output := Format(input, config) + + expected := " Column B Column C\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestColumnWidthCalculator(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "Longer than A | Longer than B | Longer than C", + "short | short | short", + } + + config := DefaultConfig() + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "Longer than A Longer than B Longer than C\n" + expected += "short short short" + + if output != expected { + printableProof := fmt.Sprintf("\nGot: %+q", output) + printableProof += fmt.Sprintf("\nExpected: %+q", expected) + t.Fatalf("\n%s", printableProof) + } +} + +func TestColumnWidthCalculatorNonASCII(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "⌘⌘⌘⌘⌘⌘⌘⌘ | Longer than B | Longer than C", + "short | short | short", + } + + config := DefaultConfig() + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "⌘⌘⌘⌘⌘⌘⌘⌘ Longer than B Longer than C\n" + expected += "short short short" + + if output != expected { + printableProof := fmt.Sprintf("\nGot: %+q", output) + printableProof += fmt.Sprintf("\nExpected: %+q", expected) + t.Fatalf("\n%s", printableProof) + } +} + +func BenchmarkColumnWidthCalculator(b *testing.B) { + // Generate the input + input := []string{ + "UUID A | UUID B | UUID C | Column D | Column E", + } + + format := "%s|%s|%s|%s" + short := "short" + + uuid := func() string { + buf := make([]byte, 16) + if _, err := crand.Read(buf); err != nil { + panic(fmt.Errorf("failed to read random bytes: %v", err)) + } + + return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x", + buf[0:4], + buf[4:6], + buf[6:8], + buf[8:10], + buf[10:16]) + } + + for i := 0; i < 1000; i++ { + l := fmt.Sprintf(format, uuid()[:8], uuid()[:12], uuid(), short, short) + input = append(input, l) + } + + config := DefaultConfig() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + Format(input, config) + } +} + +func TestVariedInputSpacing(t *testing.T) { + input := []string{ + "Column A |Column B| Column C", + "x|y| z", + } + + config := DefaultConfig() + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestUnmatchedColumnCounts(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "Value A | Value B", + "Value A | Value B | Value C | Value D", + } + + config := DefaultConfig() + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "Value A Value B\n" + expected += "Value A Value B Value C Value D" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestAlternateDelimiter(t *testing.T) { + input := []string{ + "Column | A % Column | B % Column | C", + "Value A % Value B % Value C", + } + + config := DefaultConfig() + config.Delim = "%" + output := Format(input, config) + + expected := "Column | A Column | B Column | C\n" + expected += "Value A Value B Value C" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestAlternateSpacingString(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "x | y | z", + } + + config := DefaultConfig() + config.Glue = " " + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestSimpleFormat(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "x | y | z", + } + + output := SimpleFormat(input) + + expected := "Column A Column B Column C\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestAlternatePrefixString(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "x | y | z", + } + + config := DefaultConfig() + config.Prefix = " " + output := Format(input, config) + + expected := " Column A Column B Column C\n" + expected += " x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestEmptyFieldReplacement(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "x | | z", + } + + config := DefaultConfig() + config.Empty = "" + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "x z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestEmptyConfigValues(t *testing.T) { + input := []string{ + "Column A | Column B | Column C", + "x | y | z", + } + + config := Config{} + output := Format(input, &config) + + expected := "Column A Column B Column C\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + +func TestMergeConfig(t *testing.T) { + conf1 := &Config{Delim: "a", Glue: "a", Prefix: "a", Empty: "a"} + conf2 := &Config{Delim: "b", Glue: "b", Prefix: "b", Empty: "b"} + conf3 := &Config{Delim: "c", Prefix: "c"} + + m := MergeConfig(conf1, conf2) + if m.Delim != "b" || m.Glue != "b" || m.Prefix != "b" || m.Empty != "b" { + t.Fatalf("bad: %#v", m) + } + + m = MergeConfig(conf1, conf3) + if m.Delim != "c" || m.Glue != "a" || m.Prefix != "c" || m.Empty != "a" { + t.Fatalf("bad: %#v", m) + } + + m = MergeConfig(conf1, nil) + if m.Delim != "a" || m.Glue != "a" || m.Prefix != "a" || m.Empty != "a" { + t.Fatalf("bad: %#v", m) + } + + m = MergeConfig(conf1, &Config{}) + if m.Delim != "a" || m.Glue != "a" || m.Prefix != "a" || m.Empty != "a" { + t.Fatalf("bad: %#v", m) + } +} diff --git a/plugins/config/subcommands/default b/plugins/config/subcommands/default deleted file mode 100755 index 2a266cc81..000000000 --- a/plugins/config/subcommands/default +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/config/functions" - -config_main_cmd() { - declare desc="print config vars for app via command line" - local cmd="config" - config_all "$@" -} - -config_main_cmd "$@" diff --git a/plugins/config/subcommands/get b/plugins/config/subcommands/get deleted file mode 100755 index 6a0822ab4..000000000 --- a/plugins/config/subcommands/get +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/config/functions" - -config_get_cmd() { - declare desc="get specified config vars from app via command line" - local cmd="config:get" - config_get "$@" -} - -config_get_cmd "$@" diff --git a/plugins/config/subcommands/set b/plugins/config/subcommands/set deleted file mode 100755 index 54253d3aa..000000000 --- a/plugins/config/subcommands/set +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/config/functions" - -config_set_cmd() { - declare desc="set specified config vars for app via command line" - local cmd="config:set" - config_set "$@" -} - -config_set_cmd "$@" diff --git a/plugins/config/subcommands/unset b/plugins/config/subcommands/unset deleted file mode 100755 index abb864d03..000000000 --- a/plugins/config/subcommands/unset +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_AVAILABLE_PATH/config/functions" - -config_unset_cmd() { - declare desc="unset specified config vars for app via command line" - local cmd="config:unset" - config_unset "$@" -} - -config_unset_cmd "$@" From 7d865ccb1f888f24ecb923a277cc254986477272 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Tue, 17 Jan 2017 19:55:46 -0500 Subject: [PATCH 05/51] [config] refactor makefile --- plugins/config/Makefile | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/plugins/config/Makefile b/plugins/config/Makefile index 8718ca0bc..2daae1719 100644 --- a/plugins/config/Makefile +++ b/plugins/config/Makefile @@ -1,32 +1,25 @@ -.PHONY: build-in-docker build clean +include ../../common.mk -GO_REPO_ROOT ?= /go/src/github.com/dokku/dokku -BUILD_IMAGE ?= golang:1.7.1 +GO_ARGS ?= -a + +SUBCOMMANDS = subcommands/export subcommands/get subcommands/set subcommands/unset build-in-docker: clean docker run --rm \ -v $$PWD/../..:$(GO_REPO_ROOT) \ -w $(GO_REPO_ROOT)/plugins/config \ $(BUILD_IMAGE) \ - bash -c "make build" || exit $$? + bash -c "GO_ARGS='$(GO_ARGS)' make build" || exit $$? build: commands subcommands -subcommands: subcommands/export subcommands/set subcommands/get subcommands/unset commands: **/**/commands.go - go build -a -o commands src/commands/commands.go + go build $(GO_ARGS) -o commands src/commands/commands.go -subcommands/export: **/**/**/export.go - go build -a -o subcommands/export src/subcommands/export/export.go +subcommands: $(SUBCOMMANDS) -subcommands/set: **/**/**/set.go - go build -a -o subcommands/set src/subcommands/set/set.go - -subcommands/get: **/**/**/get.go - go build -a -o subcommands/get src/subcommands/get/get.go - -subcommands/unset: **/**/**/unset.go - go build -a -o subcommands/unset src/subcommands/unset/unset.go +subcommands/%: src/subcommands/*/%.go + go build $(GO_ARGS) -o $@ $< clean: rm -rf commands subcommands From 14757ca92583e4f10a54363ede3e65ad8148491a Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 19 Jan 2017 00:02:56 -0500 Subject: [PATCH 06/51] [config] change file format to match bash exports got rid of nonstandard escaping/quoting so that the disk file is a strict subset of valid bash exports --- plugins/config/src/configenv/configenv.go | 6 +-- .../config/src/configenv/configenv_test.go | 41 +++++++++++++------ plugins/config/src/configenv/parsing.go | 38 +++++++++-------- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index d449c7bbe..65ae6884a 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -37,13 +37,13 @@ func (e *Env) stringWithEntryPrefix(prefix string) string { sort.Strings(keys) entries := make([]string, len(keys)) for i, k := range keys { - entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, escape(e.env[k])) + entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, singleQuoteEscape(e.env[k])) } return strings.Join(entries, "\n") } -func escape(value string) string { //TODO: improve this - return strings.Replace(value, "'", "\\'", -1) +func singleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\''aped' + return strings.Replace(value, "'", "'\\''", -1) } //NewFromTarget creates an env from the given target. Tareget is either "--global" or an app name diff --git a/plugins/config/src/configenv/configenv_test.go b/plugins/config/src/configenv/configenv_test.go index d0b3cbf02..a948f865f 100644 --- a/plugins/config/src/configenv/configenv_test.go +++ b/plugins/config/src/configenv/configenv_test.go @@ -6,27 +6,42 @@ import ( . "github.com/onsi/gomega" ) -func TestBasic(t *testing.T) { +func TestExportfileRoundtrip(t *testing.T) { RegisterTestingT(t) - env, err := NewFromString("HI=ho") + env, err := NewFromString("HI='ho'\nFoo='Bar'\n\nBaz='BOFF'") Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "ho"))) - Expect(env.String()).To(Equal("HI='ho'")) + Expect(env.Map()).To(Equal(pairs("Baz", "BOFF", "Foo", "Bar", "HI", "ho"))) + Expect(env.String()).To(Equal("Baz='BOFF'\nFoo='Bar'\nHI='ho'")) - env, err = NewFromString("HI=h\\o\\'") + env, err = NewFromString("\n export HI='h\\\no\\'\n") Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "h\\o'"))) - Expect(env.String()).To(Equal("HI='h\\o\\''")) + Expect(env.Map()).To(Equal(pairs("HI", "h\\\no\\"))) + Expect(env.String()).To(Equal("HI='h\\\no\\'")) - env, err = NewFromString("HI=ho\nFOO=bar\n\n") + env, err = NewFromString("HI='ho'\nFOO=''\\''\nBAR='\\'''\\'''") Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "ho", "FOO", "bar"))) - Expect(env.String()).To(Equal("FOO='bar'\nHI='ho'")) + Expect(env.Map()).To(Equal(pairs("HI", "ho", "FOO", "'\nBAR=''"))) + Expect(env.String()).To(Equal("FOO=''\\''\nBAR='\\'''\\'''\nHI='ho'")) - env, err = NewFromString("HI='ho\n'\n\nFOO='bar\\'\nbaz'") + env, err = NewFromString("HI='ho\n'\n\nFOO='=bar\"\n'\\''\nbaz'") Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "ho\n", "FOO", "bar'\nbaz"))) - Expect(env.String()).To(Equal("FOO='bar\\'\nbaz'\nHI='ho\n'")) + Expect(env.Map()).To(Equal(pairs("HI", "ho\n", "FOO", "=bar\"\n'\nbaz"))) + Expect(env.String()).To(Equal("FOO='=bar\"\n'\\''\nbaz'\nHI='ho\n'")) +} + +func TestExportfileErrors(t *testing.T) { + RegisterTestingT(t) + _, err := NewFromString("FOO=bar") //values must be quoted + Expect(err).To(HaveOccurred()) + + _, err = NewFromString("FOO='bar\\''") //single quotes are not escaped this way + Expect(err).To(HaveOccurred()) + + _, err = NewFromString("F\nOO='bar'") //keys cannot have embedded newlines + Expect(err).To(HaveOccurred()) + + _, err = NewFromString("FOO='bar' ") //no trailing content + Expect(err).To(HaveOccurred()) } func pairs(vars ...string) map[string]string { diff --git a/plugins/config/src/configenv/parsing.go b/plugins/config/src/configenv/parsing.go index bf539156b..3b30f6d9c 100644 --- a/plugins/config/src/configenv/parsing.go +++ b/plugins/config/src/configenv/parsing.go @@ -24,15 +24,13 @@ func parseEnv(name string, filename string) (*Env, error) { } func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, error) { - //The file format looks like KEY='VALUE'. Eait pair is terminated with a newline. - //All characters are valid for VALUE without escaping - //but one: the single quote and backslash. To represent a single quote emit \'. To represent - //a literal backslash emit \\. In any other case, the backslash is interpreted literally. - //KEY='value - //one' - //KEY='val\\ue\\'' etc.. + //The file format looks like: `KEY='VALUE'`. Eait pair is terminated with a newline. + //All characters are valid for VALUE without escaping but one: the single quote. + //A single quote literal is represented by ending the quoting + //emitting a \' and resuming the quoting. So for VALUE to be "don't care": + //`KEY='don'\''t care'` + //The advantage of this is that it's easily parsed and sourced by bash correctly. - //TODO: We might want to rework this to use shell-style quoting for single quotes buffered := bufio.NewReader(reader) const ( StateKey = iota @@ -50,9 +48,11 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e switch char { case ' ': if buffer.String() == "export" { - buffer.Truncate(0) //so we can read exportfiles too + buffer.Truncate(0) //so we can read exportfiles as well as envfiles + } else if buffer.Len() == 0 { + continue //leading spaced are allowed but not encouraged } else { - return nil, errors.New("Env keys cannot have spaces") + return nil, errors.New("keys cannot have spaces") } case '=': key = buffer.String() @@ -60,7 +60,7 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e state = StateValue case '\n': if buffer.Len() > 0 { - return nil, errors.New("Invalid newline after: " + buffer.String()) + return nil, errors.New("keys cannot contain newlines") } default: buffer.WriteRune(char) @@ -75,7 +75,7 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e quoted = !quoted } case '\n': - if escaped || quoted { + if quoted { buffer.WriteRune(char) } else { state = StateKey @@ -86,33 +86,35 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e quoted = false } case '\\': - if escaped { + if quoted { buffer.WriteRune(char) escaped = false } else { escaped = true } default: + if !quoted { + return nil, errors.New("unquoted or unbalanced value for " + key) + } if escaped { - buffer.WriteRune('\\') - escaped = false + return nil, errors.New("invalid escape for " + key) } buffer.WriteRune(char) } } } if escaped { - return nil, errors.New("Unterminated escape") + return nil, errors.New("unterminated escape at end") } if quoted { - return nil, errors.New("Unterminated quote") + return nil, errors.New("unterminated quote at end") } if state == StateValue { env.Set(key, buffer.String()) } else { if buffer.Len() > 0 { - return nil, errors.New("Invalid trailing content: " + buffer.String()) + return nil, errors.New("invalid trailing content: '" + buffer.String() + "'") } } return env, nil From dc90a889d7aad16cdab02eaa9d5bb9a0d67222a7 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 19 Jan 2017 00:21:36 -0500 Subject: [PATCH 07/51] [config] add config:set --encoded (base64) You can now set b64encoded values. They are stored decoded, this is so dokku over ssh can set odd config values ``` config:set --encoded KEY=BASE_64_VALUE ``` --- plugins/config/src/commands/commands.go | 2 +- plugins/config/src/subcommands/set/set.go | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/plugins/config/src/commands/commands.go b/plugins/config/src/commands/commands.go index d0a04dbf3..edbbadc3c 100644 --- a/plugins/config/src/commands/commands.go +++ b/plugins/config/src/commands/commands.go @@ -21,7 +21,7 @@ Additional commands:` helpContent = ` config:get (|--global) KEY, Display a global or app-specific config value - config:set (|--global) [--no-restart] KEY1=VALUE1 [KEY2=VALUE2 ...], Set one or more config vars + config:set (|--global) [--encoded] [--no-restart] KEY1=VALUE1 [KEY2=VALUE2 ...], Set one or more config vars config:unset (|--global) KEY1 [KEY2 ...], Unset one or more config vars config:export (|--global) [--envfile], Export a global or app environment ` diff --git a/plugins/config/src/subcommands/set/set.go b/plugins/config/src/subcommands/set/set.go index 5e0b38dab..b01be629f 100644 --- a/plugins/config/src/subcommands/set/set.go +++ b/plugins/config/src/subcommands/set/set.go @@ -1,11 +1,11 @@ package main import ( + "encoding/base64" "flag" - - "strings" - + "fmt" "os" + "strings" common "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config/src/configenv" @@ -15,6 +15,7 @@ import ( func main() { args := flag.NewFlagSet("config:set", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") + encoded := args.Bool("encoded", false, "--encoded: interpret VALUEs as base64") noRestart := args.Bool("no-restart", false, "--no-restart: no restart") args.Parse(os.Args[2:]) shouldRestart := !*global && !*noRestart @@ -43,7 +44,15 @@ func main() { if len(parts) == 1 { common.LogFail("Invalid env pair: " + e) } - env.Set(parts[0], parts[1]) + key, value := parts[0], parts[1] + if *encoded { + decoded, err := base64.StdEncoding.DecodeString(value) + if err != nil { + common.LogFail(fmt.Sprintf("%s for key '%s'", err.Error(), key)) + } + value = string(decoded) + } + env.Set(key, value) changed = true } From 416b4996706d570b05ce91ef6e34ef117a78053c Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 21 Jan 2017 21:17:52 -0500 Subject: [PATCH 08/51] [nginx-proxy] remove trailing spaces from PROXY_PORT_MAP this plugin used to rely on the fact that config supressed trailing spaces --- plugins/nginx-vhosts/functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/nginx-vhosts/functions b/plugins/nginx-vhosts/functions index 4792550df..35521f66a 100755 --- a/plugins/nginx-vhosts/functions +++ b/plugins/nginx-vhosts/functions @@ -302,6 +302,8 @@ nginx_build_config() { SPDY_SUPPORTED="$(is_spdy_enabled "$NGINX_VERSION")" HTTP2_SUPPORTED="$(is_http2_enabled "$NGINX_VERSION")" + PROXY_PORT_MAP=$(echo "$PROXY_PORT_MAP" | xargs) # trailing spaces mess up default template + eval "$(config_export app "$APP")" local SIGIL_PARAMS=(-f $NGINX_TEMPLATE APP="$APP" DOKKU_ROOT="$DOKKU_ROOT" NOSSL_SERVER_NAME="$NOSSL_SERVER_NAME" From 21b091ab14f2659329686dab9e5eb9ec431baea4 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 21 Jan 2017 21:19:15 -0500 Subject: [PATCH 09/51] [config] update test-deploy-config to include odd characters --- tests/apps/config/check_deploy | 3 ++- tests/apps/config/post-deploy | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/apps/config/check_deploy b/tests/apps/config/check_deploy index d21cdf62f..46d7828d9 100755 --- a/tests/apps/config/check_deploy +++ b/tests/apps/config/check_deploy @@ -12,4 +12,5 @@ test_url () { base_url="$1" test_url / "config-test" -test_url /hello "Hello world" +test_url /hello "Hello + 'world'" \ No newline at end of file diff --git a/tests/apps/config/post-deploy b/tests/apps/config/post-deploy index 04f5a11d5..61cda9ea4 100755 --- a/tests/apps/config/post-deploy +++ b/tests/apps/config/post-deploy @@ -4,6 +4,7 @@ set -ex; REMOTE=$1 REPO=$2 -# setting config variables with spaces is broken at the moment. ref: 820 -# ssh "$REMOTE" config:set "$REPO" CONFTEST=config-test HELLO=\"Hello world\" -dokku config:set "$REPO" CONFTEST=config-test HELLO="Hello world" +GREETING=$(printf "Hello + 'world'" | base64) +ssh "$REMOTE" config:set "$REPO" CONFTEST=config-test +ssh "$REMOTE" config:set --encoded "$REPO" HELLO="$GREETING" From dd8e1fdf45601ecad173342ee8a814f33e6cd69e Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 21 Jan 2017 21:20:03 -0500 Subject: [PATCH 10/51] [config] no longer reenter dokku in config/functions --- plugins/config/functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/config/functions b/plugins/config/functions index 582d49018..0a60f5fec 100755 --- a/plugins/config/functions +++ b/plugins/config/functions @@ -9,21 +9,21 @@ config_export() { if [[ $CONFIG_TYPE == "global" ]]; then APP="--global" fi - dokku config:export "$APP" + $PLUGIN_AVAILABLE_PATH/config/subcommands/export config:export "$APP" return 0 } config_get() { declare desc="get value of given config var" - dokku config:get "$@" + $PLUGIN_AVAILABLE_PATH/config/subcommands/get config:get "$@" } config_set() { declare desc="set value of given config var" - dokku config:set "$@" + $PLUGIN_AVAILABLE_PATH/config/subcommands/set config:set "$@" } config_unset() { declare desc="set value of given config var" - dokku config:unset "$@" + $PLUGIN_AVAILABLE_PATH/config/subcommands/unset config:unset "$@" } From dfa241d6a980a72b6ec20446d3db2011c194aeed Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 23 Jan 2017 22:56:33 -0500 Subject: [PATCH 11/51] [dockerfile] test new config stuff --- tests/apps/dockerfile/app.json | 4 ++++ tests/apps/dockerfile/check_deploy | 18 +++++++++++++++++- tests/apps/dockerfile/index.js | 4 ++++ tests/apps/dockerfile/post-deploy | 9 +++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100755 tests/apps/dockerfile/post-deploy diff --git a/tests/apps/dockerfile/app.json b/tests/apps/dockerfile/app.json index d3d57e1fe..80fd1ed17 100644 --- a/tests/apps/dockerfile/app.json +++ b/tests/apps/dockerfile/app.json @@ -23,6 +23,10 @@ "WEB_CONCURRENCY": { "description": "The number of processes to run.", "generator": "echo 5" + }, + "FROM_APP_JSON": { + "description": "The number of processes to run.", + "value": "'foo\nbar'" } }, "image": "gliderlabs/herokuish", diff --git a/tests/apps/dockerfile/check_deploy b/tests/apps/dockerfile/check_deploy index b652fb8f6..6c9b953ce 100755 --- a/tests/apps/dockerfile/check_deploy +++ b/tests/apps/dockerfile/check_deploy @@ -1,2 +1,18 @@ #!/usr/bin/env bash -set -e; output="$(curl -s -S "$1")"; echo "$output"; test "$output" == "Hello World!" +set -e; + +test_url () { + path="$1" + expected_output="$2" + url="$base_url$path" + output="$(curl -s -S "$url:3000")" + echo "$output" + test "$output" == "$expected_output" +} + +base_url="$1" +test_url / "Hello World!" +test_url /env/FROM_APP_JSON "'foo +bar'" +test_url /env/HELLO "Hello + 'world' " \ No newline at end of file diff --git a/tests/apps/dockerfile/index.js b/tests/apps/dockerfile/index.js index be6855ac3..027e20aa4 100644 --- a/tests/apps/dockerfile/index.js +++ b/tests/apps/dockerfile/index.js @@ -8,6 +8,10 @@ app.get('/', function(request, response) { response.send('Hello World!') }) +app.get('/env/:key', function(request, response) { + response.send(process.env[request.params["key"]]) +}) + app.listen(app.get('port'), function() { console.log("Node app is running at localhost:" + app.get('port')) }) diff --git a/tests/apps/dockerfile/post-deploy b/tests/apps/dockerfile/post-deploy new file mode 100755 index 000000000..973188fb0 --- /dev/null +++ b/tests/apps/dockerfile/post-deploy @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ex; + +REMOTE=$1 +REPO=$2 + +GREETING=$(printf "Hello + 'world' " | base64) +ssh "$REMOTE" config:set --encoded "$REPO" HELLO="$GREETING" From 73e85b7df52d7ff102e3bcdf0ce7c0a14066725f Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 30 Jan 2017 22:20:39 -0500 Subject: [PATCH 12/51] [config] preliminary Dockerfile support --- .../00_dokku-standard/exec-app-json-scripts | 7 +++-- plugins/00_dokku-standard/pre-deploy | 8 +++--- plugins/00_dokku-standard/subcommands/run | 5 +++- plugins/common/functions | 17 ++++++++---- plugins/config/docker-args-deploy | 27 +++++++++++++++---- plugins/config/functions | 11 ++++++++ plugins/config/src/configenv/configenv.go | 5 ++-- .../config/src/subcommands/export/export.go | 8 ++++++ plugins/config/src/subcommands/get/get.go | 8 +++++- tests/apps/dockerfile/check_deploy | 7 ++--- 10 files changed, 79 insertions(+), 24 deletions(-) diff --git a/plugins/00_dokku-standard/exec-app-json-scripts b/plugins/00_dokku-standard/exec-app-json-scripts index 7bb3e5809..e90457b2c 100755 --- a/plugins/00_dokku-standard/exec-app-json-scripts +++ b/plugins/00_dokku-standard/exec-app-json-scripts @@ -55,8 +55,11 @@ execute_script() { # strip --restart args from DOCKER_ARGS local DOCKER_ARGS=$(sed -e "s/--restart=[[:graph:]]\+[[:blank:]]\?//g" <<< "$DOCKER_ARGS") - # shellcheck disable=SC2086 - local id=$(docker run "$DOKKU_GLOBAL_RUN_ARGS" -e DOKKU_TRACE="$DOKKU_TRACE" --label=dokku_phase_script="${PHASE_SCRIPT_KEY}" -d -v "$CACHE_DIR:/cache" $DOCKER_ARGS "$IMAGE" /bin/bash -c "$COMMAND") + # eval args as array to respect escapes + declare -a ARG_ARRAY + eval ARG_ARRAY=($DOCKER_ARGS) + + local id=$(docker run "$DOKKU_GLOBAL_RUN_ARGS" -e DOKKU_TRACE="$DOKKU_TRACE" --label=dokku_phase_script="${PHASE_SCRIPT_KEY}" -d -v "$CACHE_DIR:/cache" "${ARG_ARRAY[@]}" "$IMAGE" /bin/bash -c "$COMMAND") if test "$(docker wait "$id")" -eq 0; then dokku_container_log_verbose_quiet "$id" if [[ "$PHASE_SCRIPT_KEY" != "postdeploy" ]];then diff --git a/plugins/00_dokku-standard/pre-deploy b/plugins/00_dokku-standard/pre-deploy index c78257140..ec8b1ba30 100755 --- a/plugins/00_dokku-standard/pre-deploy +++ b/plugins/00_dokku-standard/pre-deploy @@ -15,8 +15,9 @@ exec_app_json_scripts() { app_user_pre_deploy_trigger() { local desc="app user pre-deploy trigger" - local APP="$1"; local IMAGE_TAG="$2"; local IMAGE; local DOCKER_ARGS + local APP="$1"; local IMAGE_TAG="$2"; local IMAGE; local DOCKER ARGS local DOKKU_APP_TYPE DOKKU_APP_USER APP_PATHS CONTAINER_PATHS + declare -a ARG_ARRAY IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") @@ -27,12 +28,13 @@ app_user_pre_deploy_trigger() { if [[ -n "$APP_PATHS" ]]; then CONTAINER_PATHS=$(echo "$APP_PATHS" | awk -F ':' '{ print $2 }' | xargs) - DOCKER_ARGS=$(: | plugn trigger docker-args-deploy "$APP" "$IMAGE_TAG" | xargs) + DOCKER_ARGS=$(: | plugn trigger docker-args-deploy "$APP" "$IMAGE_TAG") + eval ARG_ARRAY=($DOCKER_ARGS) fi if [[ "$DOKKU_APP_TYPE" == "herokuish" ]] && [[ -n "$CONTAINER_PATHS" ]]; then # shellcheck disable=SC2086 - docker run $DOKKU_GLOBAL_RUN_ARGS $DOCKER_ARGS $IMAGE /bin/bash -c "find $CONTAINER_PATHS -not -user $DOKKU_APP_USER -print0 | xargs -0 -r chown -R $DOKKU_APP_USER" || true + docker run $DOKKU_GLOBAL_RUN_ARGS "${ARG_ARRAY[@]}" $IMAGE /bin/bash -c "find $CONTAINER_PATHS -not -user $DOKKU_APP_USER -print0 | xargs -0 -r chown -R $DOKKU_APP_USER" || true fi } diff --git a/plugins/00_dokku-standard/subcommands/run b/plugins/00_dokku-standard/subcommands/run index 1a844d8fe..9f5f6aa41 100755 --- a/plugins/00_dokku-standard/subcommands/run +++ b/plugins/00_dokku-standard/subcommands/run @@ -22,6 +22,9 @@ dokku_run_cmd() { local DOCKER_ARGS=$(: | plugn trigger docker-args-run "$APP" "$IMAGE_TAG") [[ "$DOKKU_TRACE" ]] && local DOCKER_ARGS+=" -e TRACE=true " + declare -a ARG_ARRAY + eval ARG_ARRAY=($DOCKER_ARGS) + local DOKKU_RUN_OPTS=" -e DYNO=run.$RANDOM" if [[ "$DOKKU_RM_CONTAINER" ]]; then DOKKU_RUN_OPTS+=" --rm" @@ -44,7 +47,7 @@ dokku_run_cmd() { fi # shellcheck disable=SC2086 - docker run $DOKKU_GLOBAL_RUN_ARGS $DOKKU_RUN_OPTS $DOCKER_ARGS $IMAGE $EXEC_CMD "$@" + docker run $DOKKU_GLOBAL_RUN_ARGS $DOKKU_RUN_OPTS "${ARG_ARRAY[@]}" $IMAGE $EXEC_CMD "$@" } dokku_run_cmd "$@" diff --git a/plugins/common/functions b/plugins/common/functions index c9f28250a..eb306ebf7 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -458,8 +458,10 @@ dokku_build() { local DOCKER_ARGS=$(: | plugn trigger docker-args-build "$APP" "$IMAGE_SOURCE_TYPE") [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" -e TRACE=true " + declare -a ARG_ARRAY + eval ARG_ARRAY=($DOCKER_ARGS) # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -v $CACHE_DIR:/cache -e CACHE_PATH=/cache $DOCKER_ARGS $IMAGE /build) + local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -v $CACHE_DIR:/cache -e CACHE_PATH=/cache "${ARG_ARRAY[@]}" $IMAGE /build) docker attach "$id" test "$(docker wait "$id")" -eq 0 docker commit "$id" "$IMAGE" > /dev/null @@ -483,9 +485,11 @@ dokku_build() { local DOCKER_ARGS=$(: | plugn trigger docker-args-build "$APP" "$IMAGE_SOURCE_TYPE") # strip --volume and -v args from DOCKER_ARGS local DOCKER_ARGS=$(sed -e "s/--volume=[[:graph:]]\+[[:blank:]]\?//g" -e "s/-v[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" <<< "$DOCKER_ARGS") + declare -a ARG_ARRAY + eval ARG_ARRAY=($DOCKER_ARGS) # shellcheck disable=SC2086 - docker build $DOCKER_ARGS $DOKKU_DOCKER_BUILD_OPTS -t $IMAGE . + docker build "${ARG_ARRAY[@]}" $DOKKU_DOCKER_BUILD_OPTS -t $IMAGE . plugn trigger post-build-dockerfile "$APP" ;; @@ -583,6 +587,9 @@ dokku_deploy_cmd() { DOCKER_ARGS+=" -e DYNO=$PROC_TYPE.$CONTAINER_INDEX " [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" -e TRACE=true " + declare -a ARG_ARRAY + eval ARG_ARRAY=($DOCKER_ARGS) + [[ -n "$DOKKU_HEROKUISH" ]] && local START_CMD="/start $PROC_TYPE" if [[ -z "$DOKKU_HEROKUISH" ]]; then @@ -609,7 +616,7 @@ dokku_deploy_cmd() { fi if [[ "$DOKKU_IS_APP_PROXY_ENABLED" == "true" ]]; then # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -e PORT=$port $DOCKER_ARGS $IMAGE $START_CMD) + local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -e PORT=$port "${ARG_ARRAY[@]}" $IMAGE $START_CMD) local ipaddr=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$id") # Docker < 1.9 compatibility if [[ -z $ipaddr ]]; then @@ -617,13 +624,13 @@ dokku_deploy_cmd() { fi else # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d $DOKKU_DOCKER_PORT_ARGS -e PORT=$port $DOCKER_ARGS $IMAGE $START_CMD) + local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d $DOKKU_DOCKER_PORT_ARGS -e PORT=$port "${ARG_ARRAY[@]}" $IMAGE $START_CMD) local port=$(docker port "$id" "$port" | sed 's/[0-9.]*://') local ipaddr=127.0.0.1 fi else # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d $DOCKER_ARGS $IMAGE $START_CMD) + local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d "${ARG_ARRAY[@]}" $IMAGE $START_CMD) fi kill_new() { diff --git a/plugins/config/docker-args-deploy b/plugins/config/docker-args-deploy index cd89cc14d..366d2dd7d 100755 --- a/plugins/config/docker-args-deploy +++ b/plugins/config/docker-args-deploy @@ -7,15 +7,32 @@ config_docker_args() { declare desc="config docker-args plugin trigger" local trigger="$0 config_docker_args" local STDIN=$(cat); local APP="$1"; local IMAGE_TAG="$2"; local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") - local DOCKERFILE_ENV_FILE="$DOKKU_ROOT/$APP/DOCKERFILE_ENV_FILE" verify_app_name "$APP" if ! is_image_herokuish_based "$IMAGE"; then - > "$DOCKERFILE_ENV_FILE" - config_export global | sed -e "s:^export ::g" -e "s:=':=:g" -e "s:'$::g" > "$DOCKERFILE_ENV_FILE" - config_export app "$APP" | sed -e "s:^export ::g" -e "s:=':=:g" -e "s:'$::g" >> "$DOCKERFILE_ENV_FILE" + shopt -s extglob + local ENV_ARGS + declare -A ENV_ARGS + for key in $(config_keys global); do + VAR="$(config_get --global "$key")" + printf -vVAR '%q' "${VAR%%*( )}" + ENV_ARGS[$key]="$VAR" + done + for key in $(config_keys app "$APP"); do + VAR="$(config_get "$APP" "$key")" + printf -vVAR '%q' "${VAR%%*( )}" + ENV_ARGS[$key]="$VAR" + done + shopt -u extglob + local ALL_ENV_ARGS + declare -a ALL_ENV_ARGS + for key in "${!ENV_ARGS[@]}"; do + local ARG="$key=${ENV_ARGS[$key]}" + ALL_ENV_ARGS+=("--env") + ALL_ENV_ARGS+=("$ARG") + done - echo -n "$STDIN --env-file=$DOCKERFILE_ENV_FILE" + echo -n "$STDIN ${ALL_ENV_ARGS[@]}" else echo -n "$STDIN" fi diff --git a/plugins/config/functions b/plugins/config/functions index 0a60f5fec..0dda7d76e 100755 --- a/plugins/config/functions +++ b/plugins/config/functions @@ -13,6 +13,17 @@ config_export() { return 0 } +config_keys() { + declare desc="returns keys for app or global" + local CONFIG_TYPE="$1" + local APP="$2" + if [[ $CONFIG_TYPE == "global" ]]; then + APP="--global" + fi + $PLUGIN_AVAILABLE_PATH/config/subcommands/export config:export --keys "$APP" + return 0 +} + config_get() { declare desc="get value of given config var" $PLUGIN_AVAILABLE_PATH/config/subcommands/get config:get "$@" diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index 65ae6884a..7fdf53d7d 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -37,12 +37,13 @@ func (e *Env) stringWithEntryPrefix(prefix string) string { sort.Strings(keys) entries := make([]string, len(keys)) for i, k := range keys { - entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, singleQuoteEscape(e.env[k])) + entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, SingleQuoteEscape(e.env[k])) } return strings.Join(entries, "\n") } -func singleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\''aped' +//SingleQuoteEscape escapes the value as if it were shell-quoted in single quotes +func SingleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\''aped' return strings.Replace(value, "'", "'\\''", -1) } diff --git a/plugins/config/src/subcommands/export/export.go b/plugins/config/src/subcommands/export/export.go index 52feae877..9253dc88f 100644 --- a/plugins/config/src/subcommands/export/export.go +++ b/plugins/config/src/subcommands/export/export.go @@ -13,6 +13,7 @@ import ( func main() { args := flag.NewFlagSet("config:export", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") + keys := args.Bool("keys", false, "--keys: export keys only") envfile := args.Bool("envfile", false, "--envfile: export as envfile rather than bash exports") args.Parse(os.Args[2:]) @@ -32,6 +33,13 @@ func main() { if err != nil { common.LogFail(err.Error()) } + if *keys { + for _, k := range env.Keys() { + fmt.Println(k) + } + return + } + if *envfile { fmt.Println(env.EnvfileString()) } else { diff --git a/plugins/config/src/subcommands/get/get.go b/plugins/config/src/subcommands/get/get.go index 3c57eea71..321ed856b 100644 --- a/plugins/config/src/subcommands/get/get.go +++ b/plugins/config/src/subcommands/get/get.go @@ -13,6 +13,7 @@ import ( func main() { args := flag.NewFlagSet("config:get", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") + quoted := args.Bool("quoted", false, "--quoted: get the value quoted") args.Parse(os.Args[2:]) appName := args.Arg(0) nextArg := 0 @@ -35,5 +36,10 @@ func main() { common.LogFail(fmt.Sprintf("Unexpected argument(s): %v", args.Args()[nextArg+1:])) } key := args.Arg(nextArg) - fmt.Printf("%s", env.GetDefault(key, "")) + value := env.GetDefault(key, "") + if *quoted { + fmt.Printf("'%s'", configenv.SingleQuoteEscape(value)) + } else { + fmt.Printf("%s", value) + } } diff --git a/tests/apps/dockerfile/check_deploy b/tests/apps/dockerfile/check_deploy index 6c9b953ce..4fa457760 100755 --- a/tests/apps/dockerfile/check_deploy +++ b/tests/apps/dockerfile/check_deploy @@ -4,15 +4,12 @@ set -e; test_url () { path="$1" expected_output="$2" - url="$base_url$path" - output="$(curl -s -S "$url:3000")" - echo "$output" + url="${base_url}:3000${path}" + output="$(curl -s -S "$url")" test "$output" == "$expected_output" } base_url="$1" test_url / "Hello World!" -test_url /env/FROM_APP_JSON "'foo -bar'" test_url /env/HELLO "Hello 'world' " \ No newline at end of file From 686428a41a1bbd97ea184765085236d06c272c32 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 18 Feb 2017 15:56:31 -0500 Subject: [PATCH 13/51] [config] lint fixes --- .../00_dokku-standard/exec-app-json-scripts | 2 +- plugins/00_dokku-standard/pre-deploy | 2 +- plugins/00_dokku-standard/subcommands/run | 2 +- plugins/common/functions | 6 ++--- plugins/config/functions | 27 +++++++++++-------- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/plugins/00_dokku-standard/exec-app-json-scripts b/plugins/00_dokku-standard/exec-app-json-scripts index e90457b2c..8077e7a65 100755 --- a/plugins/00_dokku-standard/exec-app-json-scripts +++ b/plugins/00_dokku-standard/exec-app-json-scripts @@ -57,7 +57,7 @@ execute_script() { # eval args as array to respect escapes declare -a ARG_ARRAY - eval ARG_ARRAY=($DOCKER_ARGS) + eval "ARG_ARRAY=($DOCKER_ARGS)" local id=$(docker run "$DOKKU_GLOBAL_RUN_ARGS" -e DOKKU_TRACE="$DOKKU_TRACE" --label=dokku_phase_script="${PHASE_SCRIPT_KEY}" -d -v "$CACHE_DIR:/cache" "${ARG_ARRAY[@]}" "$IMAGE" /bin/bash -c "$COMMAND") if test "$(docker wait "$id")" -eq 0; then diff --git a/plugins/00_dokku-standard/pre-deploy b/plugins/00_dokku-standard/pre-deploy index ec8b1ba30..c21ccc7fc 100755 --- a/plugins/00_dokku-standard/pre-deploy +++ b/plugins/00_dokku-standard/pre-deploy @@ -29,7 +29,7 @@ app_user_pre_deploy_trigger() { if [[ -n "$APP_PATHS" ]]; then CONTAINER_PATHS=$(echo "$APP_PATHS" | awk -F ':' '{ print $2 }' | xargs) DOCKER_ARGS=$(: | plugn trigger docker-args-deploy "$APP" "$IMAGE_TAG") - eval ARG_ARRAY=($DOCKER_ARGS) + eval "ARG_ARRAY=($DOCKER_ARGS)" fi if [[ "$DOKKU_APP_TYPE" == "herokuish" ]] && [[ -n "$CONTAINER_PATHS" ]]; then diff --git a/plugins/00_dokku-standard/subcommands/run b/plugins/00_dokku-standard/subcommands/run index 9f5f6aa41..a0dc3cbd2 100755 --- a/plugins/00_dokku-standard/subcommands/run +++ b/plugins/00_dokku-standard/subcommands/run @@ -23,7 +23,7 @@ dokku_run_cmd() { [[ "$DOKKU_TRACE" ]] && local DOCKER_ARGS+=" -e TRACE=true " declare -a ARG_ARRAY - eval ARG_ARRAY=($DOCKER_ARGS) + eval "ARG_ARRAY=($DOCKER_ARGS)" local DOKKU_RUN_OPTS=" -e DYNO=run.$RANDOM" if [[ "$DOKKU_RM_CONTAINER" ]]; then diff --git a/plugins/common/functions b/plugins/common/functions index eb306ebf7..1db8dba99 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -459,7 +459,7 @@ dokku_build() { local DOCKER_ARGS=$(: | plugn trigger docker-args-build "$APP" "$IMAGE_SOURCE_TYPE") [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" -e TRACE=true " declare -a ARG_ARRAY - eval ARG_ARRAY=($DOCKER_ARGS) + eval "ARG_ARRAY=($DOCKER_ARGS)" # shellcheck disable=SC2086 local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -v $CACHE_DIR:/cache -e CACHE_PATH=/cache "${ARG_ARRAY[@]}" $IMAGE /build) docker attach "$id" @@ -486,7 +486,7 @@ dokku_build() { # strip --volume and -v args from DOCKER_ARGS local DOCKER_ARGS=$(sed -e "s/--volume=[[:graph:]]\+[[:blank:]]\?//g" -e "s/-v[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" <<< "$DOCKER_ARGS") declare -a ARG_ARRAY - eval ARG_ARRAY=($DOCKER_ARGS) + eval "ARG_ARRAY=($DOCKER_ARGS)" # shellcheck disable=SC2086 docker build "${ARG_ARRAY[@]}" $DOKKU_DOCKER_BUILD_OPTS -t $IMAGE . @@ -588,7 +588,7 @@ dokku_deploy_cmd() { [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" -e TRACE=true " declare -a ARG_ARRAY - eval ARG_ARRAY=($DOCKER_ARGS) + eval "ARG_ARRAY=($DOCKER_ARGS)" [[ -n "$DOKKU_HEROKUISH" ]] && local START_CMD="/start $PROC_TYPE" diff --git a/plugins/config/functions b/plugins/config/functions index 0dda7d76e..f2787a170 100755 --- a/plugins/config/functions +++ b/plugins/config/functions @@ -2,39 +2,44 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +config_sub(){ + declare desc="executes a config subcommand" + local name="$1" + shift + "$PLUGIN_AVAILABLE_PATH/config/subcommands/$name" "config:$name" "$@" +} + config_export() { declare desc="returns export command for config variable of specified type (app/global)" local CONFIG_TYPE="$1" - local APP="$2" + shift + local APP="$1" if [[ $CONFIG_TYPE == "global" ]]; then APP="--global" + else + shift fi - $PLUGIN_AVAILABLE_PATH/config/subcommands/export config:export "$APP" + config_sub export "$@" "$APP" return 0 } config_keys() { declare desc="returns keys for app or global" - local CONFIG_TYPE="$1" - local APP="$2" - if [[ $CONFIG_TYPE == "global" ]]; then - APP="--global" - fi - $PLUGIN_AVAILABLE_PATH/config/subcommands/export config:export --keys "$APP" + config_export "$@" "--keys" return 0 } config_get() { declare desc="get value of given config var" - $PLUGIN_AVAILABLE_PATH/config/subcommands/get config:get "$@" + config_sub get "$@" } config_set() { declare desc="set value of given config var" - $PLUGIN_AVAILABLE_PATH/config/subcommands/set config:set "$@" + config_sub set "$@" } config_unset() { declare desc="set value of given config var" - $PLUGIN_AVAILABLE_PATH/config/subcommands/unset config:unset "$@" + config_sub unset "$@" } From fd27d90303161e7e209992b08a4736b0d18f096d Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 18 Feb 2017 15:58:32 -0500 Subject: [PATCH 14/51] [config] config:export --merge, --prefix, & --suffix --- plugins/config/docker-args-deploy | 25 +----------- plugins/config/src/configenv/configenv.go | 40 +++++++++++++++---- .../config/src/configenv/configenv_test.go | 15 +++++++ .../config/src/subcommands/export/export.go | 28 +++++++++++-- 4 files changed, 73 insertions(+), 35 deletions(-) diff --git a/plugins/config/docker-args-deploy b/plugins/config/docker-args-deploy index 366d2dd7d..55b9f5026 100755 --- a/plugins/config/docker-args-deploy +++ b/plugins/config/docker-args-deploy @@ -10,29 +10,8 @@ config_docker_args() { verify_app_name "$APP" if ! is_image_herokuish_based "$IMAGE"; then - shopt -s extglob - local ENV_ARGS - declare -A ENV_ARGS - for key in $(config_keys global); do - VAR="$(config_get --global "$key")" - printf -vVAR '%q' "${VAR%%*( )}" - ENV_ARGS[$key]="$VAR" - done - for key in $(config_keys app "$APP"); do - VAR="$(config_get "$APP" "$key")" - printf -vVAR '%q' "${VAR%%*( )}" - ENV_ARGS[$key]="$VAR" - done - shopt -u extglob - local ALL_ENV_ARGS - declare -a ALL_ENV_ARGS - for key in "${!ENV_ARGS[@]}"; do - local ARG="$key=${ENV_ARGS[$key]}" - ALL_ENV_ARGS+=("--env") - ALL_ENV_ARGS+=("$ARG") - done - - echo -n "$STDIN ${ALL_ENV_ARGS[@]}" + local ENV_ARGS="$(config_export app $APP --prefix="-e" --separator=" " --merged)" + echo -n "$STDIN $ENV_ARGS" else echo -n "$STDIN" fi diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index 7fdf53d7d..070ea0007 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -24,22 +24,23 @@ func (e *Env) String() string { //EnvfileString returns the contents of this Env in ENVFILE format func (e *Env) EnvfileString() string { - return e.stringWithEntryPrefix("") + return e.StringWithPrefixAndSeparator("", "\n") } //ExportfileString returns the contents of this Env as bash exports func (e *Env) ExportfileString() string { - return e.stringWithEntryPrefix("export ") + return e.StringWithPrefixAndSeparator("export ", "\n") } -func (e *Env) stringWithEntryPrefix(prefix string) string { +//StringWithPrefixAndSeparator makes a string of the environment +// with the given prefix and separator for each entry +func (e *Env) StringWithPrefixAndSeparator(prefix string, separator string) string { keys := e.Keys() - sort.Strings(keys) entries := make([]string, len(keys)) for i, k := range keys { entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, SingleQuoteEscape(e.env[k])) } - return strings.Join(entries, "\n") + return strings.Join(entries, separator) } //SingleQuoteEscape escapes the value as if it were shell-quoted in single quotes @@ -50,13 +51,23 @@ func SingleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\'' //NewFromTarget creates an env from the given target. Tareget is either "--global" or an app name func NewFromTarget(target string) (*Env, error) { if target == "--global" { - return parseEnv("global", getGlobalFile()) + return LoadGlobal() } - appfile, err := getAppFile(target) + return LoadApp(target) +} + +//LoadApp loads an environment for the given app +func LoadApp(appName string) (*Env, error) { + appfile, err := getAppFile(appName) if err != nil { return nil, err } - return parseEnv(target, appfile) + return parseEnv(appName, appfile) +} + +//LoadGlobal loads the global environmen +func LoadGlobal() (*Env, error) { + return parseEnv("global", getGlobalFile()) } //NewFromString creates an env from the given ENVFILE contents representation @@ -64,6 +75,13 @@ func NewFromString(rep string) (*Env, error) { return parseEnvFromReader("", "", strings.NewReader(rep)) } +//Merge merges the given environment on top of the reciever +func (e *Env) Merge(other *Env) { + for _, k := range other.Keys() { + e.Set(k, other.GetDefault(k, "")) + } +} + //Set an environment variable func (e *Env) Set(key string, value string) { e.env[key] = value @@ -80,6 +98,7 @@ func (e *Env) Keys() []string { for k := range e.env { keys = append(keys, k) } + sort.Strings(keys) return keys } @@ -108,6 +127,11 @@ func (e *Env) GetBoolDefault(key string, defaultValue bool) bool { return v != "0" } +//Len return the number of items in this environment +func (e *Env) Len() int { + return len(e.env) +} + //Map return the Env as a map func (e *Env) Map() map[string]string { return e.env diff --git a/plugins/config/src/configenv/configenv_test.go b/plugins/config/src/configenv/configenv_test.go index a948f865f..16b3e3db0 100644 --- a/plugins/config/src/configenv/configenv_test.go +++ b/plugins/config/src/configenv/configenv_test.go @@ -44,6 +44,21 @@ func TestExportfileErrors(t *testing.T) { Expect(err).To(HaveOccurred()) } +func TestMerge(t *testing.T) { + RegisterTestingT(t) + e, _ := NewFromString("FOO='bar'") + e2, _ := NewFromString("BAR='baz'") + e.Merge(e2) + Expect(e.Map()).To(Equal(pairs("BAR", "baz", "FOO", "bar"))) +} + +func TestArrayExport(t *testing.T) { + RegisterTestingT(t) + e, _ := NewFromString("BAR='BAZ'\nFOO='b'\\''ar '") + Expect(e.StringWithPrefixAndSeparator("", " ")).To(Equal("BAR='BAZ' FOO='b'\\''ar '")) + Expect(e.StringWithPrefixAndSeparator("-e", " ")).To(Equal("-eBAR='BAZ' -eFOO='b'\\''ar '")) +} + func pairs(vars ...string) map[string]string { res := map[string]string{} var i = 0 diff --git a/plugins/config/src/subcommands/export/export.go b/plugins/config/src/subcommands/export/export.go index 9253dc88f..d28a9b2f4 100644 --- a/plugins/config/src/subcommands/export/export.go +++ b/plugins/config/src/subcommands/export/export.go @@ -11,10 +11,14 @@ import ( // print the environment to stdout func main() { + const defaultPrefix = "export " args := flag.NewFlagSet("config:export", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") + merged := args.Bool("merged", false, "--merged: merge app environment and global environment") keys := args.Bool("keys", false, "--keys: export keys only") - envfile := args.Bool("envfile", false, "--envfile: export as envfile rather than bash exports") + envfile := args.Bool("envfile", false, "--envfile: export as envfile rather than bash exports (--prefix='')") + prefix := args.String("prefix", "export ", "--prefix: prefix") + separator := args.String("separator", "\n", "--separator: separator") args.Parse(os.Args[2:]) appName := args.Arg(0) @@ -26,6 +30,9 @@ func main() { if appName != "" { common.LogFail("Trailing argument: " + appName) } + if *merged == true { + common.LogFail("Only app environments can be merged") + } appName = "--global" } @@ -33,6 +40,16 @@ func main() { if err != nil { common.LogFail(err.Error()) } + + if *merged { + global, err := configenv.LoadGlobal() + if err != nil { + common.LogFail(err.Error()) + } + global.Merge(env) + env = global + } + if *keys { for _, k := range env.Keys() { fmt.Println(k) @@ -41,8 +58,11 @@ func main() { } if *envfile { - fmt.Println(env.EnvfileString()) - } else { - fmt.Println(env.ExportfileString()) + if *prefix != defaultPrefix { + common.LogFail("--prefix and --envfile cannot both be given") + return + } + *prefix = "" } + fmt.Println(env.StringWithPrefixAndSeparator(*prefix, *separator)) } From c20fbfccf680e67f30e712f9292775c0b1dad850 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 22 Apr 2017 15:58:03 -0400 Subject: [PATCH 15/51] [config] support for env bundles --- plugins/build-env/pre-build-buildpack | 35 +++++----------- plugins/config/src/configenv/configenv.go | 40 ++++++++++++++++--- plugins/config/src/configenv/parsing.go | 2 +- .../config/src/subcommands/export/export.go | 17 +++++++- 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/plugins/build-env/pre-build-buildpack b/plugins/build-env/pre-build-buildpack index 57d32917d..33863fbbf 100755 --- a/plugins/build-env/pre-build-buildpack +++ b/plugins/build-env/pre-build-buildpack @@ -6,36 +6,23 @@ source "$PLUGIN_AVAILABLE_PATH/config/functions" build_env_pre_build_buildpack() { declare desc="build-env pre-build-buildpack plugin trigger" local trigger="build_env_pre_build_buildpack" - local APP="$1"; local IMAGE=$(get_app_image_name "$APP"); local BUILD_ENV="" + local APP="$1"; local IMAGE=$(get_app_image_name "$APP"); verify_app_name "$APP" [[ -z $(config_get --global CURL_CONNECT_TIMEOUT) ]] && config_set --global CURL_CONNECT_TIMEOUT=90 [[ -z $(config_get --global CURL_TIMEOUT) ]] && config_set --global CURL_TIMEOUT=60 - if [[ -n $(config_export global) ]]; then - local BUILD_ENV+=$'\n' - local BUILD_ENV+=$(config_export global) - local BUILD_ENV+=$'\n' - fi - if [[ -n $(config_export app "$APP") ]]; then - local BUILD_ENV+=$'\n' - local BUILD_ENV+=$(config_export app "$APP") - local BUILD_ENV+=$'\n' - fi + dokku_log_info1 "Adding BUILD_ENV to build environment..." + # create build env files for use in buildpacks like this: + # https://github.com/niteoweb/heroku-buildpack-buildout/blob/5879fa3418f7d8e079f1aa5816ba1adde73f4948/bin/compile#L34 + local id=$(config_export app "$APP" --merged --bundle | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "mkdir -p /tmp/env; cat | tar -x -C /tmp/env") + test "$(docker wait "$id")" -eq 0 + docker commit "$id" "$IMAGE" > /dev/null - if [[ ! -z "$BUILD_ENV" ]]; then - dokku_log_info1 "Adding BUILD_ENV to build environment..." - # create build env files for use in buildpacks like this: - # https://github.com/niteoweb/heroku-buildpack-buildout/blob/5879fa3418f7d8e079f1aa5816ba1adde73f4948/bin/compile#L34 - local id=$(echo $BUILD_ENV | sed -e 's@export @@g' -e 's@\\n@ @g' | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "mkdir -p /tmp/env; for ENV_VAR in $(cat); do echo \$ENV_VAR | cut -d'=' -f2- > /tmp/env/\$(echo \$ENV_VAR | cut -d'=' -f1); done " ) - test "$(docker wait "$id")" -eq 0 - docker commit "$id" "$IMAGE" > /dev/null - - # create build env for 'old style' buildpacks and dokku plugins - local id=$(echo -e "$BUILD_ENV" | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "cat >> /app/.env") - test "$(docker wait "$id")" -eq 0 - docker commit "$id" "$IMAGE" > /dev/null - fi + # create build env for 'old style' buildpacks and dokku plugins + local id=$(config_export app "$APP" --envfile --merged --escape-newlines | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "cat >> /app/.env") + test "$(docker wait "$id")" -eq 0 + docker commit "$id" "$IMAGE" > /dev/null } build_env_pre_build_buildpack "$@" diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index 070ea0007..db6c79ecb 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -3,19 +3,23 @@ package configenv import ( "errors" "fmt" + "io" "os" "path/filepath" "sort" "strings" + "archive/tar" + common "github.com/dokku/dokku/plugins/common" ) //Env is a representation for global or app environment type Env struct { - name string - env map[string]string - filename string + name string + env map[string]string + filename string + EscapeNewlines bool } func (e *Env) String() string { @@ -38,7 +42,11 @@ func (e *Env) StringWithPrefixAndSeparator(prefix string, separator string) stri keys := e.Keys() entries := make([]string, len(keys)) for i, k := range keys { - entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, SingleQuoteEscape(e.env[k])) + v := SingleQuoteEscape(e.env[k]) + if e.EscapeNewlines { + v = strings.Replace(v, "\n", "'$'\\n''", -1) + } + entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, v) } return strings.Join(entries, separator) } @@ -48,7 +56,29 @@ func SingleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\'' return strings.Replace(value, "'", "'\\''", -1) } -//NewFromTarget creates an env from the given target. Tareget is either "--global" or an app name +//ExportBundle writes a tarfile of the environmnet to the given io.Writer. +// for every environment variable there is a file with the variable's key +// with its content set to the variable's value +func (e *Env) ExportBundle(dest io.Writer) error { + tarfile := tar.NewWriter(dest) + defer tarfile.Close() + + for _, k := range e.Keys() { + val, _ := e.Get(k) + valbin := []byte(val) + + header := &tar.Header{ + Name: k, + Mode: 0600, + Size: int64(len(valbin)), + } + tarfile.WriteHeader(header) + tarfile.Write(valbin) + } + return nil +} + +//NewFromTarget creates an env from the given target. Target is either "--global" or an app name func NewFromTarget(target string) (*Env, error) { if target == "--global" { return LoadGlobal() diff --git a/plugins/config/src/configenv/parsing.go b/plugins/config/src/configenv/parsing.go index 3b30f6d9c..1c8feda37 100644 --- a/plugins/config/src/configenv/parsing.go +++ b/plugins/config/src/configenv/parsing.go @@ -36,7 +36,7 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e StateKey = iota StateValue = iota ) - env := &Env{name, map[string]string{}, filename} + env := &Env{name, map[string]string{}, filename, false} var buffer bytes.Buffer var state = StateKey var quoted = false diff --git a/plugins/config/src/subcommands/export/export.go b/plugins/config/src/subcommands/export/export.go index d28a9b2f4..1a75d51c8 100644 --- a/plugins/config/src/subcommands/export/export.go +++ b/plugins/config/src/subcommands/export/export.go @@ -12,13 +12,16 @@ import ( // print the environment to stdout func main() { const defaultPrefix = "export " + const defaultSeparator = "\n" args := flag.NewFlagSet("config:export", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") merged := args.Bool("merged", false, "--merged: merge app environment and global environment") keys := args.Bool("keys", false, "--keys: export keys only") + bundle := args.Bool("bundle", false, "--bundle: export as tar bundle") envfile := args.Bool("envfile", false, "--envfile: export as envfile rather than bash exports (--prefix='')") - prefix := args.String("prefix", "export ", "--prefix: prefix") - separator := args.String("separator", "\n", "--separator: separator") + prefix := args.String("prefix", defaultPrefix, "--prefix: prefix") + separator := args.String("separator", defaultSeparator, "--separator: separator") + escapeNewlines := args.Bool("escape-newlines", false, "--escape-newlines: replace literal newlines with $'\n'") args.Parse(os.Args[2:]) appName := args.Arg(0) @@ -57,6 +60,16 @@ func main() { return } + if *escapeNewlines { + env.EscapeNewlines = true + } + + if *bundle { + if *prefix != defaultPrefix || *separator != defaultSeparator || *envfile { + common.LogFail("--bundle cannot be given with --envfile, --prefix, or --separator") + } + env.ExportBundle(os.Stdout) + } if *envfile { if *prefix != defaultPrefix { common.LogFail("--prefix and --envfile cannot both be given") From 12c9536a7bf605a6a64182004adba7124e80244a Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 30 Apr 2017 22:29:58 -0400 Subject: [PATCH 16/51] [config] test python buildpack too --- tests/apps/python-flask/check_deploy | 16 +++++++++++++++- tests/apps/python-flask/hello.py | 12 +++++++++++- tests/apps/python-flask/post-deploy | 9 +++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100755 tests/apps/python-flask/post-deploy diff --git a/tests/apps/python-flask/check_deploy b/tests/apps/python-flask/check_deploy index 8c7299cc8..65408ff64 100755 --- a/tests/apps/python-flask/check_deploy +++ b/tests/apps/python-flask/check_deploy @@ -1,2 +1,16 @@ #!/usr/bin/env bash -set -e; output="$(curl -s -S "$1")"; echo "$output"; test "$output" == "python/flask" +set -e; +base_url=$1 +test_url () { + path="$1" + expected_output="$2" + url="${base_url}${path}" + output="$(curl -s -S "$url")" + echo "$url -> $output" + test "$output" == "$expected_output" +} + +base_url="$1" +test_url / "python/flask" +test_url /HELLO "Hello + 'world' " diff --git a/tests/apps/python-flask/hello.py b/tests/apps/python-flask/hello.py index 9dc2cec78..b6a5ba771 100644 --- a/tests/apps/python-flask/hello.py +++ b/tests/apps/python-flask/hello.py @@ -1,8 +1,18 @@ import os -from flask import Flask +from flask import Flask, abort app = Flask(__name__) @app.route('/') def hello(): return 'python/flask' + +@app.route('/') +def get_enc(env): + val = os.environ.get(env, None) + if val: + return val + else: + abort(404) + + diff --git a/tests/apps/python-flask/post-deploy b/tests/apps/python-flask/post-deploy new file mode 100755 index 000000000..973188fb0 --- /dev/null +++ b/tests/apps/python-flask/post-deploy @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ex; + +REMOTE=$1 +REPO=$2 + +GREETING=$(printf "Hello + 'world' " | base64) +ssh "$REMOTE" config:set --encoded "$REPO" HELLO="$GREETING" From e7e01ee8387aa0aee5d0619e3d39e3a6d274268f Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 1 May 2017 01:47:26 -0400 Subject: [PATCH 17/51] [config] fix shellcheck --- plugins/config/docker-args-deploy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/config/docker-args-deploy b/plugins/config/docker-args-deploy index 55b9f5026..b64acb23e 100755 --- a/plugins/config/docker-args-deploy +++ b/plugins/config/docker-args-deploy @@ -10,7 +10,7 @@ config_docker_args() { verify_app_name "$APP" if ! is_image_herokuish_based "$IMAGE"; then - local ENV_ARGS="$(config_export app $APP --prefix="-e" --separator=" " --merged)" + local ENV_ARGS="$(config_export app "$APP" --prefix="-e" --separator=" " --merged)" echo -n "$STDIN $ENV_ARGS" else echo -n "$STDIN" From 133fdc1f0ae5761e634d052689e92e26326cdc18 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 1 May 2017 01:48:02 -0400 Subject: [PATCH 18/51] [config] support for old export-style config files --- plugins/config/src/configenv/configenv_test.go | 11 +++++++---- plugins/config/src/configenv/parsing.go | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/plugins/config/src/configenv/configenv_test.go b/plugins/config/src/configenv/configenv_test.go index 16b3e3db0..56033f81c 100644 --- a/plugins/config/src/configenv/configenv_test.go +++ b/plugins/config/src/configenv/configenv_test.go @@ -13,11 +13,16 @@ func TestExportfileRoundtrip(t *testing.T) { Expect(env.Map()).To(Equal(pairs("Baz", "BOFF", "Foo", "Bar", "HI", "ho"))) Expect(env.String()).To(Equal("Baz='BOFF'\nFoo='Bar'\nHI='ho'")) - env, err = NewFromString("\n export HI='h\\\no\\'\n") + env, err = NewFromString("\n export HI='h\\\no\\' \n") Expect(err).NotTo(HaveOccurred()) Expect(env.Map()).To(Equal(pairs("HI", "h\\\no\\"))) Expect(env.String()).To(Equal("HI='h\\\no\\'")) + env, err = NewFromString("\n export HI=ho\n") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("HI", "ho"))) + Expect(env.String()).To(Equal("HI='ho'")) + env, err = NewFromString("HI='ho'\nFOO=''\\''\nBAR='\\'''\\'''") Expect(err).NotTo(HaveOccurred()) Expect(env.Map()).To(Equal(pairs("HI", "ho", "FOO", "'\nBAR=''"))) @@ -31,10 +36,8 @@ func TestExportfileRoundtrip(t *testing.T) { func TestExportfileErrors(t *testing.T) { RegisterTestingT(t) - _, err := NewFromString("FOO=bar") //values must be quoted - Expect(err).To(HaveOccurred()) - _, err = NewFromString("FOO='bar\\''") //single quotes are not escaped this way + _, err := NewFromString("FOO='bar\\''") //single quotes are not escaped this way Expect(err).To(HaveOccurred()) _, err = NewFromString("F\nOO='bar'") //keys cannot have embedded newlines diff --git a/plugins/config/src/configenv/parsing.go b/plugins/config/src/configenv/parsing.go index 1c8feda37..762a65358 100644 --- a/plugins/config/src/configenv/parsing.go +++ b/plugins/config/src/configenv/parsing.go @@ -6,6 +6,7 @@ import ( "errors" "io" "os" + "strings" ) func parseEnv(name string, filename string) (*Env, error) { @@ -24,7 +25,7 @@ func parseEnv(name string, filename string) (*Env, error) { } func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, error) { - //The file format looks like: `KEY='VALUE'`. Eait pair is terminated with a newline. + //The file format looks like: `KEY='VALUE'`. Each pair is terminated with a newline. //All characters are valid for VALUE without escaping but one: the single quote. //A single quote literal is represented by ending the quoting //emitting a \' and resuming the quoting. So for VALUE to be "don't care": @@ -40,6 +41,7 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e var buffer bytes.Buffer var state = StateKey var quoted = false + var exportStyle = false var escaped = false var key = "" for char, _, err := buffered.ReadRune(); err == nil; char, _, err = buffered.ReadRune() { @@ -48,6 +50,7 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e switch char { case ' ': if buffer.String() == "export" { + exportStyle = true buffer.Truncate(0) //so we can read exportfiles as well as envfiles } else if buffer.Len() == 0 { continue //leading spaced are allowed but not encouraged @@ -79,11 +82,16 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e buffer.WriteRune(char) } else { state = StateKey - env.Set(key, buffer.String()) + value := buffer.String() + if exportStyle { + value = strings.TrimSpace(value) + } + env.Set(key, value) buffer.Truncate(0) key = "" escaped = false quoted = false + exportStyle = false } case '\\': if quoted { @@ -93,8 +101,8 @@ func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, e escaped = true } default: - if !quoted { - return nil, errors.New("unquoted or unbalanced value for " + key) + if !quoted && !exportStyle { + return nil, errors.New("trailing content for value of " + key) } if escaped { return nil, errors.New("invalid escape for " + key) From 8b90d4ebf94f04fa8a9154e4b193ce76ad2d601f Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 4 May 2017 22:39:55 -0400 Subject: [PATCH 19/51] [config] move vendor to right place --- plugins/config/{src => }/glide.lock | 0 plugins/config/{src => }/glide.yaml | 0 .../{src => }/vendor/github.com/ryanuber/columnize/.travis.yml | 0 .../config/{src => }/vendor/github.com/ryanuber/columnize/LICENSE | 0 .../{src => }/vendor/github.com/ryanuber/columnize/README.md | 0 .../{src => }/vendor/github.com/ryanuber/columnize/columnize.go | 0 .../vendor/github.com/ryanuber/columnize/columnize_test.go | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename plugins/config/{src => }/glide.lock (100%) rename plugins/config/{src => }/glide.yaml (100%) rename plugins/config/{src => }/vendor/github.com/ryanuber/columnize/.travis.yml (100%) rename plugins/config/{src => }/vendor/github.com/ryanuber/columnize/LICENSE (100%) rename plugins/config/{src => }/vendor/github.com/ryanuber/columnize/README.md (100%) rename plugins/config/{src => }/vendor/github.com/ryanuber/columnize/columnize.go (100%) rename plugins/config/{src => }/vendor/github.com/ryanuber/columnize/columnize_test.go (100%) diff --git a/plugins/config/src/glide.lock b/plugins/config/glide.lock similarity index 100% rename from plugins/config/src/glide.lock rename to plugins/config/glide.lock diff --git a/plugins/config/src/glide.yaml b/plugins/config/glide.yaml similarity index 100% rename from plugins/config/src/glide.yaml rename to plugins/config/glide.yaml diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/.travis.yml b/plugins/config/vendor/github.com/ryanuber/columnize/.travis.yml similarity index 100% rename from plugins/config/src/vendor/github.com/ryanuber/columnize/.travis.yml rename to plugins/config/vendor/github.com/ryanuber/columnize/.travis.yml diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/LICENSE b/plugins/config/vendor/github.com/ryanuber/columnize/LICENSE similarity index 100% rename from plugins/config/src/vendor/github.com/ryanuber/columnize/LICENSE rename to plugins/config/vendor/github.com/ryanuber/columnize/LICENSE diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/README.md b/plugins/config/vendor/github.com/ryanuber/columnize/README.md similarity index 100% rename from plugins/config/src/vendor/github.com/ryanuber/columnize/README.md rename to plugins/config/vendor/github.com/ryanuber/columnize/README.md diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize.go b/plugins/config/vendor/github.com/ryanuber/columnize/columnize.go similarity index 100% rename from plugins/config/src/vendor/github.com/ryanuber/columnize/columnize.go rename to plugins/config/vendor/github.com/ryanuber/columnize/columnize.go diff --git a/plugins/config/src/vendor/github.com/ryanuber/columnize/columnize_test.go b/plugins/config/vendor/github.com/ryanuber/columnize/columnize_test.go similarity index 100% rename from plugins/config/src/vendor/github.com/ryanuber/columnize/columnize_test.go rename to plugins/config/vendor/github.com/ryanuber/columnize/columnize_test.go From 08aa2b9fa4e413599edfb558bb125176a6338350 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 4 May 2017 22:41:32 -0400 Subject: [PATCH 20/51] [config] improve logging and default command --- plugins/common/common.go | 29 +++++++++++++++++++ plugins/config/config.go | 26 +++++++++++++++++ plugins/config/src/commands/commands.go | 16 ++++++---- plugins/config/src/subcommands/set/set.go | 11 ++++--- plugins/config/src/subcommands/unset/unset.go | 6 ++-- 5 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 plugins/config/config.go diff --git a/plugins/common/common.go b/plugins/common/common.go index 7d0f1106e..92c9076fc 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -150,3 +150,32 @@ func PlugnTrigger(triggerName string, args ...string) (string, error) { res, err := sh.Command("plugn", "trigger", triggerName, args).Output() return string(res), err } + +// LogInfo1 is the info1 header formatter +func LogInfo1(text string) { + fmt.Fprintln(os.Stdout, fmt.Sprintf("-----> %s", text)) +} + +// LogInfo1Quiet is the info1 header formatter (with quiet option) +func LogInfo1Quiet(text string) { + if os.Getenv("DOKKU_QUIET_OUTPUT") != "" { + LogInfo1(text) + } +} + +// LogInfo2 is the info2 header formatter +func LogInfo2(text string) { + fmt.Fprintln(os.Stdout, fmt.Sprintf("=====> %s", text)) +} + +// LogInfo2Quiet is the info2 header formatter (with quiet option) +func LogInfo2Quiet(text string) { + if os.Getenv("DOKKU_QUIET_OUTPUT") != "" { + LogInfo2(text) + } +} + +// LogWarn is the warning log formatter +func LogWarn(text string) { + fmt.Fprintln(os.Stderr, fmt.Sprintf(" ! %s", text)) +} diff --git a/plugins/config/config.go b/plugins/config/config.go new file mode 100644 index 000000000..29869cc09 --- /dev/null +++ b/plugins/config/config.go @@ -0,0 +1,26 @@ +package config + +import configenv "github.com/dokku/dokku/plugins/config/src/configenv" +import columnize "github.com/ryanuber/columnize" +import "fmt" + +//GetWithDefault gets value from app config +func GetWithDefault(appName string, key string, defaultValue string) string { + env, err := configenv.NewFromTarget(appName) + if err != nil { + return defaultValue + } + return env.GetDefault(key, defaultValue) +} + +//PrettyPrintLogEntries in columns +func PrettyPrintLogEntries(prefix string, entries map[string]string) string { + colConfig := columnize.DefaultConfig() + colConfig.Prefix = prefix + colConfig.Delim = "\x00" + lines := make([]string, 0, len(entries)) + for k, v := range entries { + lines = append(lines, fmt.Sprintf("%s:\x00%s", k, v)) + } + return columnize.Format(lines, colConfig) +} diff --git a/plugins/config/src/commands/commands.go b/plugins/config/src/commands/commands.go index edbbadc3c..02c909ace 100644 --- a/plugins/config/src/commands/commands.go +++ b/plugins/config/src/commands/commands.go @@ -8,6 +8,7 @@ import ( "strings" common "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config" configenv "github.com/dokku/dokku/plugins/config/src/configenv" columnize "github.com/ryanuber/columnize" ) @@ -20,9 +21,10 @@ Display all global or app-specific config vars Additional commands:` helpContent = ` - config:get (|--global) KEY, Display a global or app-specific config value - config:set (|--global) [--encoded] [--no-restart] KEY1=VALUE1 [KEY2=VALUE2 ...], Set one or more config vars - config:unset (|--global) KEY1 [KEY2 ...], Unset one or more config vars + config (|--global), Pretty-print an app or global environment + config:get (|--global) KEY, Display a global or app-specific config value + config:set (|--global) [--encoded] [--no-restart] KEY1=VALUE1 [KEY2=VALUE2 ...], Set one or more config vars + config:unset (|--global) KEY1 [KEY2 ...], Unset one or more config vars config:export (|--global) [--envfile], Export a global or app environment ` ) @@ -33,13 +35,17 @@ func main() { cmd := flag.Arg(0) switch cmd { - case "config": + case "config", "config:show": target := flag.Arg(1) + if target == "" { + usage() + common.LogFail("Please specify an app or --global") + } env, err := configenv.NewFromTarget(target) if err != nil { common.LogFail(err.Error()) } else { - fmt.Println(env.String()) + fmt.Println(config.PrettyPrintLogEntries("", env.Map())) } case "config:help": usage() diff --git a/plugins/config/src/subcommands/set/set.go b/plugins/config/src/subcommands/set/set.go index b01be629f..f3504883b 100644 --- a/plugins/config/src/subcommands/set/set.go +++ b/plugins/config/src/subcommands/set/set.go @@ -8,6 +8,7 @@ import ( "strings" common "github.com/dokku/dokku/plugins/common" + config "github.com/dokku/dokku/plugins/config" "github.com/dokku/dokku/plugins/config/src/configenv" ) @@ -36,8 +37,8 @@ func main() { common.LogFail(err.Error()) } - var changed = false entries := args.Args()[nextArg:] + updated := make(map[string]string) for _, e := range entries { //log parts := strings.SplitN(e, "=", 2) @@ -53,17 +54,19 @@ func main() { value = string(decoded) } env.Set(key, value) - changed = true + updated[key] = value } - if changed { - //log + if len(updated) != 0 { + common.LogInfo1("Setting config vars") + fmt.Println(config.PrettyPrintLogEntries(" ", updated)) env.Write() args := append([]string{appName, "set"}, entries...) common.PlugnTrigger("post-config-update", args...) } if shouldRestart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { + common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) cmd.Execute() } diff --git a/plugins/config/src/subcommands/unset/unset.go b/plugins/config/src/subcommands/unset/unset.go index ed72cf44b..7b37e1811 100644 --- a/plugins/config/src/subcommands/unset/unset.go +++ b/plugins/config/src/subcommands/unset/unset.go @@ -4,6 +4,8 @@ import ( "flag" "os" + "fmt" + common "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config/src/configenv" ) @@ -35,19 +37,19 @@ func main() { var changed = false keys := args.Args()[nextArg:] for _, k := range keys { - //log + common.LogInfo1(fmt.Sprintf("Unsetting %s", k)) env.Unset(k) changed = true } if changed { - //log env.Write() args := append([]string{appName, "unset"}, keys...) common.PlugnTrigger("post-config-update", args...) } if shouldRestart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { + common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) cmd.Execute() } From eeb6e78fa477f945818108921cc27e64bb521ab1 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 20 May 2017 14:38:27 -0400 Subject: [PATCH 21/51] [config] move to godotenv --- plugins/build-env/pre-build-buildpack | 4 +- plugins/config/Makefile | 2 +- plugins/config/config.go | 111 +++++- plugins/config/docker-args-deploy | 2 +- plugins/config/functions | 7 +- plugins/config/glide.lock | 9 +- plugins/config/glide.yaml | 2 + plugins/config/src/commands/commands.go | 16 +- plugins/config/src/configenv/configenv.go | 93 +++-- .../config/src/configenv/configenv_test.go | 38 +- plugins/config/src/configenv/parsing.go | 129 ------- .../config/src/subcommands/bundle/bundle.go | 24 ++ .../config/src/subcommands/export/export.go | 75 +--- plugins/config/src/subcommands/get/get.go | 24 +- plugins/config/src/subcommands/keys/keys.go | 26 ++ plugins/config/src/subcommands/set/set.go | 39 +- plugins/config/src/subcommands/unset/unset.go | 45 +-- .../github.com/alexquick/godotenv/.gitignore | 1 + .../github.com/alexquick/godotenv/.travis.yml | 8 + .../github.com/alexquick/godotenv/LICENCE | 23 ++ .../github.com/alexquick/godotenv/README.md | 127 +++++++ .../alexquick/godotenv/autoload/autoload.go | 15 + .../alexquick/godotenv/cmd/godotenv/cmd.go | 54 +++ .../alexquick/godotenv/fixtures/equals.env | 2 + .../alexquick/godotenv/fixtures/exported.env | 2 + .../alexquick/godotenv/fixtures/invalid1.env | 2 + .../alexquick/godotenv/fixtures/plain.env | 7 + .../alexquick/godotenv/fixtures/quoted.env | 8 + .../github.com/alexquick/godotenv/godotenv.go | 304 +++++++++++++++ .../alexquick/godotenv/godotenv_test.go | 352 ++++++++++++++++++ 30 files changed, 1189 insertions(+), 362 deletions(-) delete mode 100644 plugins/config/src/configenv/parsing.go create mode 100644 plugins/config/src/subcommands/bundle/bundle.go create mode 100644 plugins/config/src/subcommands/keys/keys.go create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/.gitignore create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/.travis.yml create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/LICENCE create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/README.md create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/autoload/autoload.go create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/cmd/godotenv/cmd.go create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/fixtures/equals.env create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/fixtures/exported.env create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/fixtures/invalid1.env create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/fixtures/plain.env create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/fixtures/quoted.env create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go create mode 100644 plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go diff --git a/plugins/build-env/pre-build-buildpack b/plugins/build-env/pre-build-buildpack index 33863fbbf..4a6e980c1 100755 --- a/plugins/build-env/pre-build-buildpack +++ b/plugins/build-env/pre-build-buildpack @@ -15,12 +15,12 @@ build_env_pre_build_buildpack() { dokku_log_info1 "Adding BUILD_ENV to build environment..." # create build env files for use in buildpacks like this: # https://github.com/niteoweb/heroku-buildpack-buildout/blob/5879fa3418f7d8e079f1aa5816ba1adde73f4948/bin/compile#L34 - local id=$(config_export app "$APP" --merged --bundle | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "mkdir -p /tmp/env; cat | tar -x -C /tmp/env") + local id=$(config_bundle --merged "$APP" | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "mkdir -p /tmp/env; cat | tar -x -C /tmp/env") test "$(docker wait "$id")" -eq 0 docker commit "$id" "$IMAGE" > /dev/null # create build env for 'old style' buildpacks and dokku plugins - local id=$(config_export app "$APP" --envfile --merged --escape-newlines | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "cat >> /app/.env") + local id=$(config_export app "$APP" --format envfile --merged | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "cat >> /app/.env") test "$(docker wait "$id")" -eq 0 docker commit "$id" "$IMAGE" > /dev/null } diff --git a/plugins/config/Makefile b/plugins/config/Makefile index 2daae1719..4eb670252 100644 --- a/plugins/config/Makefile +++ b/plugins/config/Makefile @@ -2,7 +2,7 @@ include ../../common.mk GO_ARGS ?= -a -SUBCOMMANDS = subcommands/export subcommands/get subcommands/set subcommands/unset +SUBCOMMANDS = subcommands/export subcommands/get subcommands/set subcommands/unset subcommands/keys subcommands/bundle build-in-docker: clean docker run --rm \ diff --git a/plugins/config/config.go b/plugins/config/config.go index 29869cc09..a39cf1c56 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -1,20 +1,75 @@ package config -import configenv "github.com/dokku/dokku/plugins/config/src/configenv" -import columnize "github.com/ryanuber/columnize" -import "fmt" +import ( + "fmt" -//GetWithDefault gets value from app config + "github.com/dokku/dokku/plugins/common" + configenv "github.com/dokku/dokku/plugins/config/src/configenv" + columnize "github.com/ryanuber/columnize" +) + +//GetWithDefault gets a value from a config. If appName is empty the global config is used. func GetWithDefault(appName string, key string, defaultValue string) string { - env, err := configenv.NewFromTarget(appName) + env, err := loadConfig(appName) if err != nil { return defaultValue } return env.GetDefault(key, defaultValue) } -//PrettyPrintLogEntries in columns -func PrettyPrintLogEntries(prefix string, entries map[string]string) string { +//SetMany variables in the environment. If appName is empty the global config is used. If restart is true the app is restarted. +func SetMany(appName string, entries map[string]string, restart bool) { + global := appName == "" + env := GetConfig(appName, false) + + keys := make([]string, 0, len(entries)) + + for k, v := range entries { + env.Set(k, v) + keys = append(keys, k) + } + + if len(entries) != 0 { + common.LogInfo1("Setting config vars") + fmt.Println(PrettyPrintEnvEntries(" ", entries)) + env.Write() + args := append([]string{appName, "set"}, keys...) + common.PlugnTrigger("post-config-update", args...) + } + + if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { + common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) + cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) + cmd.Execute() + } +} + +//Unset a value in a config. If appName is empty the global config is used. If restart is true the app is restarted. +func Unset(appName string, keys []string, restart bool) { + global := appName == "" + env := GetConfig(appName, false) + var changed = false + for _, k := range keys { + common.LogInfo1(fmt.Sprintf("Unsetting %s", k)) + env.Unset(k) + changed = true + } + + if changed { + env.Write() + args := append([]string{appName, "unset"}, keys...) + common.PlugnTrigger("post-config-update", args...) + } + + if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { + common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) + cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) + cmd.Execute() + } +} + +//PrettyPrintEnvEntries in columns +func PrettyPrintEnvEntries(prefix string, entries map[string]string) string { colConfig := columnize.DefaultConfig() colConfig.Prefix = prefix colConfig.Delim = "\x00" @@ -24,3 +79,45 @@ func PrettyPrintLogEntries(prefix string, entries map[string]string) string { } return columnize.Format(lines, colConfig) } + +//GetCommonArgs extracts common positional args (appName and keys) +func GetCommonArgs(global bool, args []string) (string, []string) { + nextArg := 0 + appName := "" + if !global { + if len(args) > 0 { + appName = args[0] + } + if appName == "" { + common.LogFail("Please specify an app or --global") + } else { + nextArg++ + } + } + keys := args[nextArg:] + return appName, keys +} + +//GetConfig for the given app (global config if appName is empty). Merge with global config if merged is true. +func GetConfig(appName string, merged bool) *configenv.Env { + env, err := loadConfig(appName) + if err != nil { + common.LogFail(err.Error()) + } + if appName != "" && merged { + global, err := configenv.LoadGlobal() + if err != nil { + common.LogFail(err.Error()) + } + global.Merge(env) + return global + } + return env +} + +func loadConfig(appName string) (*configenv.Env, error) { + if appName == "" || appName == "--global" { + return configenv.LoadGlobal() + } + return configenv.LoadApp(appName) +} diff --git a/plugins/config/docker-args-deploy b/plugins/config/docker-args-deploy index b64acb23e..df8dee222 100755 --- a/plugins/config/docker-args-deploy +++ b/plugins/config/docker-args-deploy @@ -10,7 +10,7 @@ config_docker_args() { verify_app_name "$APP" if ! is_image_herokuish_based "$IMAGE"; then - local ENV_ARGS="$(config_export app "$APP" --prefix="-e" --separator=" " --merged)" + local ENV_ARGS="$(config_export app "$APP" --format docker-args --merged)" echo -n "$STDIN $ENV_ARGS" else echo -n "$STDIN" diff --git a/plugins/config/functions b/plugins/config/functions index f2787a170..9de4599c4 100755 --- a/plugins/config/functions +++ b/plugins/config/functions @@ -25,7 +25,7 @@ config_export() { config_keys() { declare desc="returns keys for app or global" - config_export "$@" "--keys" + config_keys "$@" return 0 } @@ -43,3 +43,8 @@ config_unset() { declare desc="set value of given config var" config_sub unset "$@" } + +config_bundle() { + declare desc="export tar bundle of config" + config_sub bundle "$@" +} diff --git a/plugins/config/glide.lock b/plugins/config/glide.lock index fe4a98f07..dc3e9faad 100644 --- a/plugins/config/glide.lock +++ b/plugins/config/glide.lock @@ -1,6 +1,7 @@ -hash: 1ddab5de41d1514c2722bd7e24758ad4b60bf6956eb5b9b925fa071a1427f149 -updated: 2017-01-03T17:16:50.97156327-08:00 +hash: 998d313aef819c8fa68d268d075079d281a7ccab0c3a16f6d8906b20c902f627 +updated: 2017-05-20T15:15:41.795778946-04:00 imports: +- name: github.com/alexquick/godotenv + version: 147552bee9c6196b9388f15ab03fe15721264092 - name: github.com/ryanuber/columnize - version: 0fbbb3f0e3fbdc5bae7c6cd5f6c1887ebfb76360 -testImports: [] + version: 0fbbb3f0e3fbdc5bae7c6cd5f6c1887ebfb76360 \ No newline at end of file diff --git a/plugins/config/glide.yaml b/plugins/config/glide.yaml index 3f7a96657..5e318bb53 100644 --- a/plugins/config/glide.yaml +++ b/plugins/config/glide.yaml @@ -1,3 +1,5 @@ package: . import: - package: github.com/ryanuber/columnize +- package: github.com/alexquick/godotenv + version: feature-write-dotenv diff --git a/plugins/config/src/commands/commands.go b/plugins/config/src/commands/commands.go index 02c909ace..040345e42 100644 --- a/plugins/config/src/commands/commands.go +++ b/plugins/config/src/commands/commands.go @@ -7,9 +7,7 @@ import ( "strconv" "strings" - common "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" - configenv "github.com/dokku/dokku/plugins/config/src/configenv" columnize "github.com/ryanuber/columnize" ) @@ -26,6 +24,8 @@ Additional commands:` config:set (|--global) [--encoded] [--no-restart] KEY1=VALUE1 [KEY2=VALUE2 ...], Set one or more config vars config:unset (|--global) KEY1 [KEY2 ...], Unset one or more config vars config:export (|--global) [--envfile], Export a global or app environment + config:keys ( | --global) [--merged], Show keys set in environment + config:bundle ( | --global) [--merged], Bundle environment into tarfile ` ) @@ -37,16 +37,8 @@ func main() { switch cmd { case "config", "config:show": target := flag.Arg(1) - if target == "" { - usage() - common.LogFail("Please specify an app or --global") - } - env, err := configenv.NewFromTarget(target) - if err != nil { - common.LogFail(err.Error()) - } else { - fmt.Println(config.PrettyPrintLogEntries("", env.Map())) - } + env := config.GetConfig(target, false) + fmt.Print(config.PrettyPrintEnvEntries("", env.Map())) case "config:help": usage() case "help": diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index db6c79ecb..c1abb0341 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -4,46 +4,75 @@ import ( "errors" "fmt" "io" - "os" "path/filepath" "sort" "strings" "archive/tar" + "os" + + godotenv "github.com/alexquick/godotenv" common "github.com/dokku/dokku/plugins/common" ) +type ExportFormat int + +const ( + Exports ExportFormat = iota + Envfile + DockerArgs +) + //Env is a representation for global or app environment type Env struct { - name string - env map[string]string - filename string - EscapeNewlines bool + name string + filename string + env map[string]string } func (e *Env) String() string { return e.EnvfileString() } -//EnvfileString returns the contents of this Env in ENVFILE format +func (e *Env) Export(format ExportFormat) string { + switch format { + case Exports: + return e.ExportfileString() + case Envfile: + return e.EnvfileString() + case DockerArgs: + return e.DockerArgsString() + default: + common.LogFail(fmt.Sprintf("Unknown export format: %v", format)) + return "" + } +} + +//EnvfileString returns the contents of this Env in dotenv format func (e *Env) EnvfileString() string { - return e.StringWithPrefixAndSeparator("", "\n") + rep, _ := godotenv.WriteString(e.Map()) + return rep } //ExportfileString returns the contents of this Env as bash exports func (e *Env) ExportfileString() string { - return e.StringWithPrefixAndSeparator("export ", "\n") + return e.stringWithPrefixAndSeparator("export ", "\n", true) +} + +//DockerArgsString gets the contents of this Env in the form -env=KEY=VALUE --env... +func (e *Env) DockerArgsString() string { + return e.stringWithPrefixAndSeparator("--env=", " ", true) } //StringWithPrefixAndSeparator makes a string of the environment // with the given prefix and separator for each entry -func (e *Env) StringWithPrefixAndSeparator(prefix string, separator string) string { +func (e *Env) stringWithPrefixAndSeparator(prefix string, separator string, allowNewlines bool) string { keys := e.Keys() entries := make([]string, len(keys)) for i, k := range keys { v := SingleQuoteEscape(e.env[k]) - if e.EscapeNewlines { + if !allowNewlines { v = strings.Replace(v, "\n", "'$'\\n''", -1) } entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, v) @@ -78,31 +107,45 @@ func (e *Env) ExportBundle(dest io.Writer) error { return nil } -//NewFromTarget creates an env from the given target. Target is either "--global" or an app name -func NewFromTarget(target string) (*Env, error) { - if target == "--global" { - return LoadGlobal() - } - return LoadApp(target) -} - //LoadApp loads an environment for the given app func LoadApp(appName string) (*Env, error) { appfile, err := getAppFile(appName) if err != nil { return nil, err } - return parseEnv(appName, appfile) + + if _, err := os.Stat(appfile); os.IsNotExist(err) { + _, err := os.Create(appfile) + if err != nil { + common.LogFail(err.Error()) + } + } + return loadFromFile(appName, appfile) } //LoadGlobal loads the global environmen func LoadGlobal() (*Env, error) { - return parseEnv("global", getGlobalFile()) + return loadFromFile("global", getGlobalFile()) } //NewFromString creates an env from the given ENVFILE contents representation func NewFromString(rep string) (*Env, error) { - return parseEnvFromReader("", "", strings.NewReader(rep)) + envMap, err := godotenv.ReadFromReader(strings.NewReader(rep)) + env := &Env{ + "", + "", + envMap, + } + return env, err +} +func loadFromFile(name string, filename string) (*Env, error) { + envMap, err := godotenv.Read(filename) + env := &Env{ + name, + filename, + envMap, + } + return env, err } //Merge merges the given environment on top of the reciever @@ -172,13 +215,7 @@ func (e *Env) Write() error { if e.filename == "" { return errors.New("this Env was created unbound to a file") } - file, err := os.Create(e.filename) - defer file.Close() - if err != nil { - return err - } - _, err = file.WriteString(e.ExportfileString()) - return err + return godotenv.Write(e.Map(), e.filename) } func getAppFile(appName string) (string, error) { diff --git a/plugins/config/src/configenv/configenv_test.go b/plugins/config/src/configenv/configenv_test.go index 56033f81c..9b86ae452 100644 --- a/plugins/config/src/configenv/configenv_test.go +++ b/plugins/config/src/configenv/configenv_test.go @@ -11,39 +11,29 @@ func TestExportfileRoundtrip(t *testing.T) { env, err := NewFromString("HI='ho'\nFoo='Bar'\n\nBaz='BOFF'") Expect(err).NotTo(HaveOccurred()) Expect(env.Map()).To(Equal(pairs("Baz", "BOFF", "Foo", "Bar", "HI", "ho"))) - Expect(env.String()).To(Equal("Baz='BOFF'\nFoo='Bar'\nHI='ho'")) + Expect(env.String()).To(Equal("Baz=\"BOFF\"\nFoo=\"Bar\"\nHI=\"ho\"")) - env, err = NewFromString("\n export HI='h\\\no\\' \n") + env, err = NewFromString(`export HI="h\no"`) Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "h\\\no\\"))) - Expect(env.String()).To(Equal("HI='h\\\no\\'")) + Expect(env.Map()).To(Equal(pairs("HI", "h\no"))) + Expect(env.String()).To(Equal(`HI="h\no"`)) - env, err = NewFromString("\n export HI=ho\n") - Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "ho"))) - Expect(env.String()).To(Equal("HI='ho'")) - - env, err = NewFromString("HI='ho'\nFOO=''\\''\nBAR='\\'''\\'''") + env, err = NewFromString("HI='ho'\nFOO=\"'\\nBAR=''\"") Expect(err).NotTo(HaveOccurred()) Expect(env.Map()).To(Equal(pairs("HI", "ho", "FOO", "'\nBAR=''"))) - Expect(env.String()).To(Equal("FOO=''\\''\nBAR='\\'''\\'''\nHI='ho'")) + Expect(env.String()).To(Equal("FOO=\"'\\nBAR=''\"\nHI=\"ho\"")) - env, err = NewFromString("HI='ho\n'\n\nFOO='=bar\"\n'\\''\nbaz'") + env, err = NewFromString("FOO='bar' ") Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "ho\n", "FOO", "=bar\"\n'\nbaz"))) - Expect(env.String()).To(Equal("FOO='=bar\"\n'\\''\nbaz'\nHI='ho\n'")) + Expect(env.Map()).To(Equal(pairs("FOO", "bar"))) + Expect(env.String()).To(Equal(`FOO="bar"`)) + } func TestExportfileErrors(t *testing.T) { RegisterTestingT(t) - _, err := NewFromString("FOO='bar\\''") //single quotes are not escaped this way - Expect(err).To(HaveOccurred()) - - _, err = NewFromString("F\nOO='bar'") //keys cannot have embedded newlines - Expect(err).To(HaveOccurred()) - - _, err = NewFromString("FOO='bar' ") //no trailing content + _, err := NewFromString("F\nOO='bar'") //keys cannot have embedded newlines Expect(err).To(HaveOccurred()) } @@ -57,9 +47,9 @@ func TestMerge(t *testing.T) { func TestArrayExport(t *testing.T) { RegisterTestingT(t) - e, _ := NewFromString("BAR='BAZ'\nFOO='b'\\''ar '") - Expect(e.StringWithPrefixAndSeparator("", " ")).To(Equal("BAR='BAZ' FOO='b'\\''ar '")) - Expect(e.StringWithPrefixAndSeparator("-e", " ")).To(Equal("-eBAR='BAZ' -eFOO='b'\\''ar '")) + e, _ := NewFromString("BAR='BAZ'\nFOO='b'ar '") + Expect(e.EnvfileString()).To(Equal("BAR=\"BAZ\"\nFOO=\"b'ar \"")) + Expect(e.DockerArgsString()).To(Equal("--env=BAR='BAZ' --env=FOO='b'\\''ar '")) } func pairs(vars ...string) map[string]string { diff --git a/plugins/config/src/configenv/parsing.go b/plugins/config/src/configenv/parsing.go deleted file mode 100644 index 762a65358..000000000 --- a/plugins/config/src/configenv/parsing.go +++ /dev/null @@ -1,129 +0,0 @@ -package configenv - -import ( - "bufio" - "bytes" - "errors" - "io" - "os" - "strings" -) - -func parseEnv(name string, filename string) (*Env, error) { - var file *os.File - var err error - if _, err := os.Stat(filename); os.IsNotExist(err) { - file, err = os.Create(filename) - } else { - file, err = os.Open(filename) - } - if err != nil { - return nil, err - } - defer file.Close() - return parseEnvFromReader(name, filename, file) -} - -func parseEnvFromReader(name string, filename string, reader io.Reader) (*Env, error) { - //The file format looks like: `KEY='VALUE'`. Each pair is terminated with a newline. - //All characters are valid for VALUE without escaping but one: the single quote. - //A single quote literal is represented by ending the quoting - //emitting a \' and resuming the quoting. So for VALUE to be "don't care": - //`KEY='don'\''t care'` - //The advantage of this is that it's easily parsed and sourced by bash correctly. - - buffered := bufio.NewReader(reader) - const ( - StateKey = iota - StateValue = iota - ) - env := &Env{name, map[string]string{}, filename, false} - var buffer bytes.Buffer - var state = StateKey - var quoted = false - var exportStyle = false - var escaped = false - var key = "" - for char, _, err := buffered.ReadRune(); err == nil; char, _, err = buffered.ReadRune() { - switch state { - case StateKey: - switch char { - case ' ': - if buffer.String() == "export" { - exportStyle = true - buffer.Truncate(0) //so we can read exportfiles as well as envfiles - } else if buffer.Len() == 0 { - continue //leading spaced are allowed but not encouraged - } else { - return nil, errors.New("keys cannot have spaces") - } - case '=': - key = buffer.String() - buffer.Truncate(0) - state = StateValue - case '\n': - if buffer.Len() > 0 { - return nil, errors.New("keys cannot contain newlines") - } - default: - buffer.WriteRune(char) - } - case StateValue: - switch char { - case '\'': - if escaped { - buffer.WriteRune('\'') - escaped = false - } else { - quoted = !quoted - } - case '\n': - if quoted { - buffer.WriteRune(char) - } else { - state = StateKey - value := buffer.String() - if exportStyle { - value = strings.TrimSpace(value) - } - env.Set(key, value) - buffer.Truncate(0) - key = "" - escaped = false - quoted = false - exportStyle = false - } - case '\\': - if quoted { - buffer.WriteRune(char) - escaped = false - } else { - escaped = true - } - default: - if !quoted && !exportStyle { - return nil, errors.New("trailing content for value of " + key) - } - if escaped { - return nil, errors.New("invalid escape for " + key) - } - buffer.WriteRune(char) - } - } - } - if escaped { - return nil, errors.New("unterminated escape at end") - } - if quoted { - return nil, errors.New("unterminated quote at end") - } - - if state == StateValue { - env.Set(key, buffer.String()) - } else { - if buffer.Len() > 0 { - return nil, errors.New("invalid trailing content: '" + buffer.String() + "'") - } - } - return env, nil -} diff --git a/plugins/config/src/subcommands/bundle/bundle.go b/plugins/config/src/subcommands/bundle/bundle.go new file mode 100644 index 000000000..fc3b11407 --- /dev/null +++ b/plugins/config/src/subcommands/bundle/bundle.go @@ -0,0 +1,24 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config" +) + +func main() { + args := flag.NewFlagSet("config:get", flag.ExitOnError) + global := args.Bool("global", false, "--global: use the global environment") + merged := args.Bool("merged", false, "--merged: merge app environment and global environment") + args.Parse(os.Args[2:]) + + appName, trailingArgs := config.GetCommonArgs(*global, args.Args()) + if len(trailingArgs) > 0 { + common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) + } + config := config.GetConfig(appName, *merged) + config.ExportBundle(os.Stdout) +} diff --git a/plugins/config/src/subcommands/export/export.go b/plugins/config/src/subcommands/export/export.go index 1a75d51c8..24840ebf3 100644 --- a/plugins/config/src/subcommands/export/export.go +++ b/plugins/config/src/subcommands/export/export.go @@ -6,6 +6,7 @@ import ( "os" common "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config" "github.com/dokku/dokku/plugins/config/src/configenv" ) @@ -16,66 +17,26 @@ func main() { args := flag.NewFlagSet("config:export", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") merged := args.Bool("merged", false, "--merged: merge app environment and global environment") - keys := args.Bool("keys", false, "--keys: export keys only") - bundle := args.Bool("bundle", false, "--bundle: export as tar bundle") - envfile := args.Bool("envfile", false, "--envfile: export as envfile rather than bash exports (--prefix='')") - prefix := args.String("prefix", defaultPrefix, "--prefix: prefix") - separator := args.String("separator", defaultSeparator, "--separator: separator") - escapeNewlines := args.Bool("escape-newlines", false, "--escape-newlines: replace literal newlines with $'\n'") + format := args.String("format", "exports", "--format: [ exports | envfile | docker-args ] which format to export as)") args.Parse(os.Args[2:]) - appName := args.Arg(0) - if appName == "" && !*global { - common.LogFail("Please specify an app or --global") + appName, trailingArgs := config.GetCommonArgs(*global, args.Args()) + if len(trailingArgs) > 0 { + common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) } - if *global { - if appName != "" { - common.LogFail("Trailing argument: " + appName) - } - if *merged == true { - common.LogFail("Only app environments can be merged") - } - appName = "--global" + env := config.GetConfig(appName, *merged) + exportType := configenv.Exports + switch *format { + case "exports": + exportType = configenv.Exports + case "envfile": + exportType = configenv.Envfile + case "docker-args": + exportType = configenv.DockerArgs + default: + common.LogFail(fmt.Sprintf("Unknown export format: %v", *format)) } - - env, err := configenv.NewFromTarget(appName) - if err != nil { - common.LogFail(err.Error()) - } - - if *merged { - global, err := configenv.LoadGlobal() - if err != nil { - common.LogFail(err.Error()) - } - global.Merge(env) - env = global - } - - if *keys { - for _, k := range env.Keys() { - fmt.Println(k) - } - return - } - - if *escapeNewlines { - env.EscapeNewlines = true - } - - if *bundle { - if *prefix != defaultPrefix || *separator != defaultSeparator || *envfile { - common.LogFail("--bundle cannot be given with --envfile, --prefix, or --separator") - } - env.ExportBundle(os.Stdout) - } - if *envfile { - if *prefix != defaultPrefix { - common.LogFail("--prefix and --envfile cannot both be given") - return - } - *prefix = "" - } - fmt.Println(env.StringWithPrefixAndSeparator(*prefix, *separator)) + exported := env.Export(exportType) + fmt.Println(exported) } diff --git a/plugins/config/src/subcommands/get/get.go b/plugins/config/src/subcommands/get/get.go index 321ed856b..1c6194edc 100644 --- a/plugins/config/src/subcommands/get/get.go +++ b/plugins/config/src/subcommands/get/get.go @@ -6,6 +6,7 @@ import ( "os" common "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config" "github.com/dokku/dokku/plugins/config/src/configenv" ) @@ -15,28 +16,13 @@ func main() { global := args.Bool("global", false, "--global: use the global environment") quoted := args.Bool("quoted", false, "--quoted: get the value quoted") args.Parse(os.Args[2:]) - appName := args.Arg(0) - nextArg := 0 - if appName == "" && !*global { - common.LogFail("Please specify an app or --global") - } - if *global { - appName = "--global" - } else { - nextArg = 1 + appName, keys := config.GetCommonArgs(*global, args.Args()) + if len(keys) > 1 { + common.LogFail(fmt.Sprintf("Unexpected argument(s): %v", keys[1:])) } + value := config.GetWithDefault(appName, keys[0], "") - env, err := configenv.NewFromTarget(appName) - if err != nil { - common.LogFail(err.Error()) - } - - if args.NArg() > nextArg+1 { - common.LogFail(fmt.Sprintf("Unexpected argument(s): %v", args.Args()[nextArg+1:])) - } - key := args.Arg(nextArg) - value := env.GetDefault(key, "") if *quoted { fmt.Printf("'%s'", configenv.SingleQuoteEscape(value)) } else { diff --git a/plugins/config/src/subcommands/keys/keys.go b/plugins/config/src/subcommands/keys/keys.go new file mode 100644 index 000000000..0694e4dc3 --- /dev/null +++ b/plugins/config/src/subcommands/keys/keys.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config" +) + +func main() { + args := flag.NewFlagSet("config:get", flag.ExitOnError) + global := args.Bool("global", false, "--global: use the global environment") + merged := args.Bool("merged", false, "--merged: merge app environment and global environment") + args.Parse(os.Args[2:]) + + appName, trailingArgs := config.GetCommonArgs(*global, args.Args()) + if len(trailingArgs) > 0 { + common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) + } + config := config.GetConfig(appName, *merged) + for _, k := range config.Keys() { + fmt.Println(k) + } +} diff --git a/plugins/config/src/subcommands/set/set.go b/plugins/config/src/subcommands/set/set.go index f3504883b..276b8dc79 100644 --- a/plugins/config/src/subcommands/set/set.go +++ b/plugins/config/src/subcommands/set/set.go @@ -9,7 +9,6 @@ import ( common "github.com/dokku/dokku/plugins/common" config "github.com/dokku/dokku/plugins/config" - "github.com/dokku/dokku/plugins/config/src/configenv" ) // set the given entries from the specified environment @@ -19,28 +18,10 @@ func main() { encoded := args.Bool("encoded", false, "--encoded: interpret VALUEs as base64") noRestart := args.Bool("no-restart", false, "--no-restart: no restart") args.Parse(os.Args[2:]) - shouldRestart := !*global && !*noRestart - var nextArg = 0 - appName := args.Arg(0) - if appName == "" && !*global { - common.LogFail("Please specify an app or --global") - } + appName, pairs := config.GetCommonArgs(*global, args.Args()) - if *global { - appName = "--global" - } else { - nextArg = 1 - } - - env, err := configenv.NewFromTarget(appName) - if err != nil { - common.LogFail(err.Error()) - } - - entries := args.Args()[nextArg:] updated := make(map[string]string) - for _, e := range entries { - //log + for _, e := range pairs { parts := strings.SplitN(e, "=", 2) if len(parts) == 1 { common.LogFail("Invalid env pair: " + e) @@ -53,21 +34,7 @@ func main() { } value = string(decoded) } - env.Set(key, value) updated[key] = value } - - if len(updated) != 0 { - common.LogInfo1("Setting config vars") - fmt.Println(config.PrettyPrintLogEntries(" ", updated)) - env.Write() - args := append([]string{appName, "set"}, entries...) - common.PlugnTrigger("post-config-update", args...) - } - - if shouldRestart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { - common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) - cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) - cmd.Execute() - } + config.SetMany(appName, updated, !*noRestart) } diff --git a/plugins/config/src/subcommands/unset/unset.go b/plugins/config/src/subcommands/unset/unset.go index 7b37e1811..856854612 100644 --- a/plugins/config/src/subcommands/unset/unset.go +++ b/plugins/config/src/subcommands/unset/unset.go @@ -4,10 +4,7 @@ import ( "flag" "os" - "fmt" - - common "github.com/dokku/dokku/plugins/common" - "github.com/dokku/dokku/plugins/config/src/configenv" + "github.com/dokku/dokku/plugins/config" ) //unset the given entries from the given environment @@ -15,42 +12,8 @@ func main() { args := flag.NewFlagSet("config:unset", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") noRestart := args.Bool("no-restart", false, "--no-restart: no restart") + args.Parse(os.Args[2:]) - shouldRestart := !*global && !*noRestart - var nextArg = 0 - appName := args.Arg(0) - if appName == "" && !*global { - common.LogFail("Please specify an app or --global") - } - - if *global { - appName = "--global" - } else { - nextArg = 1 - } - - env, err := configenv.NewFromTarget(appName) - if err != nil { - common.LogFail(err.Error()) - } - - var changed = false - keys := args.Args()[nextArg:] - for _, k := range keys { - common.LogInfo1(fmt.Sprintf("Unsetting %s", k)) - env.Unset(k) - changed = true - } - - if changed { - env.Write() - args := append([]string{appName, "unset"}, keys...) - common.PlugnTrigger("post-config-update", args...) - } - - if shouldRestart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { - common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) - cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) - cmd.Execute() - } + appName, keys := config.GetCommonArgs(*global, args.Args()) + config.Unset(appName, keys, !*noRestart) } diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/.gitignore b/plugins/config/vendor/github.com/alexquick/godotenv/.gitignore new file mode 100644 index 000000000..e43b0f988 --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/.travis.yml b/plugins/config/vendor/github.com/alexquick/godotenv/.travis.yml new file mode 100644 index 000000000..df65350ba --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/.travis.yml @@ -0,0 +1,8 @@ +language: go + +go: + - 1.8 + +os: + - linux + - osx diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/LICENCE b/plugins/config/vendor/github.com/alexquick/godotenv/LICENCE new file mode 100644 index 000000000..e7ddd51be --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/LICENCE @@ -0,0 +1,23 @@ +Copyright (c) 2013 John Barton + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/README.md b/plugins/config/vendor/github.com/alexquick/godotenv/README.md new file mode 100644 index 000000000..aa6442b50 --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/README.md @@ -0,0 +1,127 @@ +# GoDotEnv [![Build Status](https://travis-ci.org/joho/godotenv.svg?branch=master)](https://travis-ci.org/joho/godotenv) [![Build status](https://ci.appveyor.com/api/projects/status/9v40vnfvvgde64u4?svg=true)](https://ci.appveyor.com/project/joho/godotenv) + +A Go (golang) port of the Ruby dotenv project (which loads env vars from a .env file) + +From the original Library: + +> Storing configuration in the environment is one of the tenets of a twelve-factor app. Anything that is likely to change between deployment environments–such as resource handles for databases or credentials for external services–should be extracted from the code into environment variables. +> +> But it is not always practical to set environment variables on development machines or continuous integration servers where multiple projects are run. Dotenv load variables from a .env file into ENV when the environment is bootstrapped. + +It can be used as a library (for loading in env for your own daemons etc) or as a bin command. + +There is test coverage and CI for both linuxish and windows environments, but I make no guarantees about the bin version working on windows. + +## Installation + +As a library + +```shell +go get github.com/joho/godotenv +``` + +or if you want to use it as a bin command +```shell +go get github.com/joho/godotenv/cmd/godotenv +``` + +## Usage + +Add your application configuration to your `.env` file in the root of your project: + +```shell +S3_BUCKET=YOURS3BUCKET +SECRET_KEY=YOURSECRETKEYGOESHERE +``` + +Then in your Go app you can do something like + +```go +package main + +import ( + "github.com/joho/godotenv" + "log" + "os" +) + +func main() { + err := godotenv.Load() + if err != nil { + log.Fatal("Error loading .env file") + } + + s3Bucket := os.Getenv("S3_BUCKET") + secretKey := os.Getenv("SECRET_KEY") + + // now do something with s3 or whatever +} +``` + +If you're even lazier than that, you can just take advantage of the autoload package which will read in `.env` on import + +```go +import _ "github.com/joho/godotenv/autoload" +``` + +While `.env` in the project root is the default, you don't have to be constrained, both examples below are 100% legit + +```go +_ = godotenv.Load("somerandomfile") +_ = godotenv.Load("filenumberone.env", "filenumbertwo.env") +``` + +If you want to be really fancy with your env file you can do comments and exports (below is a valid env file) + +```shell +# I am a comment and that is OK +SOME_VAR=someval +FOO=BAR # comments at line end are OK too +export BAR=BAZ +``` + +Or finally you can do YAML(ish) style + +```yaml +FOO: bar +BAR: baz +``` + +as a final aside, if you don't want godotenv munging your env you can just get a map back instead + +```go +var myEnv map[string]string +myEnv, err := godotenv.Read() + +s3Bucket := myEnv["S3_BUCKET"] +``` + +### Command Mode + +Assuming you've installed the command as above and you've got `$GOPATH/bin` in your `$PATH` + +``` +godotenv -f /some/path/to/.env some_command with some args +``` + +If you don't specify `-f` it will fall back on the default of loading `.env` in `PWD` + +## Contributing + +Contributions are most welcome! The parser itself is pretty stupidly naive and I wouldn't be surprised if it breaks with edge cases. + +*code changes without tests will not be accepted* + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Added some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +## CI + +Linux: [![wercker status](https://app.wercker.com/status/507594c2ec7e60f19403a568dfea0f78/m "wercker status")](https://app.wercker.com/project/bykey/507594c2ec7e60f19403a568dfea0f78) Windows: [![Build status](https://ci.appveyor.com/api/projects/status/9v40vnfvvgde64u4)](https://ci.appveyor.com/project/joho/godotenv) + +## Who? + +The original library [dotenv](https://github.com/bkeepers/dotenv) was written by [Brandon Keepers](http://opensoul.org/), and this port was done by [John Barton](http://whoisjohnbarton.com) based off the tests/fixtures in the original library. diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/autoload/autoload.go b/plugins/config/vendor/github.com/alexquick/godotenv/autoload/autoload.go new file mode 100644 index 000000000..fbcd2bdf8 --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/autoload/autoload.go @@ -0,0 +1,15 @@ +package autoload + +/* + You can just read the .env file on import just by doing + + import _ "github.com/joho/godotenv/autoload" + + And bob's your mother's brother +*/ + +import "github.com/joho/godotenv" + +func init() { + godotenv.Load() +} diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/cmd/godotenv/cmd.go b/plugins/config/vendor/github.com/alexquick/godotenv/cmd/godotenv/cmd.go new file mode 100644 index 000000000..04a9f6497 --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/cmd/godotenv/cmd.go @@ -0,0 +1,54 @@ +package main + +import ( + "flag" + "fmt" + "log" + + "strings" + + "github.com/joho/godotenv" +) + +func main() { + var showHelp bool + flag.BoolVar(&showHelp, "h", false, "show help") + var rawEnvFilenames string + flag.StringVar(&rawEnvFilenames, "f", "", "comma separated paths to .env files") + + flag.Parse() + + usage := ` +Run a process with a env setup from a .env file + +godotenv [-f ENV_FILE_PATHS] COMMAND_ARGS + +ENV_FILE_PATHS: comma separated paths to .env files +COMMAND_ARGS: command and args you want to run + +example + godotenv -f /path/to/something/.env,/another/path/.env fortune +` + // if no args or -h flag + // print usage and return + args := flag.Args() + if showHelp || len(args) == 0 { + fmt.Println(usage) + return + } + + // load env + var envFilenames []string + if rawEnvFilenames != "" { + envFilenames = strings.Split(rawEnvFilenames, ",") + } + + // take rest of args and "exec" them + cmd := args[0] + cmdArgs := args[1:] + + err := godotenv.Exec(envFilenames, cmd, cmdArgs) + if err != nil { + log.Fatal(err) + } +} diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/equals.env b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/equals.env new file mode 100644 index 000000000..594c5328a --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/equals.env @@ -0,0 +1,2 @@ +export OPTION_A='postgres://localhost:5432/database?sslmode=disable' + diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/exported.env b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/exported.env new file mode 100644 index 000000000..5821377c7 --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/exported.env @@ -0,0 +1,2 @@ +export OPTION_A=2 +export OPTION_B='\n' diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/invalid1.env b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/invalid1.env new file mode 100644 index 000000000..38f7e0e8b --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/invalid1.env @@ -0,0 +1,2 @@ +INVALID LINE +foo=bar diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/plain.env b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/plain.env new file mode 100644 index 000000000..43f7e44cc --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/plain.env @@ -0,0 +1,7 @@ +OPTION_A=1 +OPTION_B=2 +OPTION_C= 3 +OPTION_D =4 +OPTION_E = 5 +OPTION_F = +OPTION_G= \ No newline at end of file diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/quoted.env b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/quoted.env new file mode 100644 index 000000000..a03ce249b --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/quoted.env @@ -0,0 +1,8 @@ +OPTION_A='1' +OPTION_B='2' +OPTION_C='' +OPTION_D='\n' +OPTION_E="1" +OPTION_F="2" +OPTION_G="" +OPTION_H="\n" diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go new file mode 100644 index 000000000..e56663427 --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go @@ -0,0 +1,304 @@ +// Package godotenv is a go port of the ruby dotenv library (https://github.com/bkeepers/dotenv) +// +// Examples/readme can be found on the github page at https://github.com/joho/godotenv +// +// The TL;DR is that you make a .env file that looks something like +// +// SOME_ENV_VAR=somevalue +// +// and then in your go code you can call +// +// godotenv.Load() +// +// and all the env vars declared in .env will be avaiable through os.Getenv("SOME_ENV_VAR") +package godotenv + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "sort" + "strings" +) + +// Load will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main) +// +// If you call Load without any args it will default to loading .env in the current path +// +// You can otherwise tell it which files to load (there can be more than one) like +// +// godotenv.Load("fileone", "filetwo") +// +// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults +func Load(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, false) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Overload will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main) +// +// If you call Overload without any args it will default to loading .env in the current path +// +// You can otherwise tell it which files to load (there can be more than one) like +// +// godotenv.Overload("fileone", "filetwo") +// +// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars. +func Overload(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, true) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Read all env (with same file loading semantics as Load) but return values as +// a map rather than automatically writing values into env +func Read(filenames ...string) (envMap map[string]string, err error) { + filenames = filenamesOrDefault(filenames) + envMap = make(map[string]string) + + for _, filename := range filenames { + individualEnvMap, individualErr := readFile(filename) + + if individualErr != nil { + err = individualErr + return // return early on a spazout + } + + for key, value := range individualEnvMap { + envMap[key] = value + } + } + + return +} + +// Exec loads env vars from the specified filenames (empty map falls back to default) +// then executes the cmd specified. +// +// Simply hooks up os.Stdin/err/out to the command and calls Run() +// +// If you want more fine grained control over your command it's recommended +// that you use `Load()` or `Read()` and the `os/exec` package yourself. +func Exec(filenames []string, cmd string, cmdArgs []string) error { + Load(filenames...) + + command := exec.Command(cmd, cmdArgs...) + command.Stdin = os.Stdin + command.Stdout = os.Stdout + command.Stderr = os.Stderr + return command.Run() +} + +// Write serializes the given environment and writes it to a file +func Write(envMap map[string]string, filename string) error { + content, error := WriteString(envMap) + if error != nil { + return error + } + file, error := os.Create(filename) + if error != nil { + return error + } + _, err := file.WriteString(content) + return err +} + +// WriteString outputs the given environment as a dotenv-formatted environment file. +// +// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. +func WriteString(envMap map[string]string) (string, error) { + lines := make([]string, 0, len(envMap)) + for k, v := range envMap { + lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) + } + sort.Strings(lines) + return strings.Join(lines, "\n"), nil +} + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{".env"} + } + return filenames +} + +func loadFile(filename string, overload bool) error { + envMap, err := readFile(filename) + if err != nil { + return err + } + + currentEnv := map[string]bool{} + rawEnv := os.Environ() + for _, rawEnvLine := range rawEnv { + key := strings.Split(rawEnvLine, "=")[0] + currentEnv[key] = true + } + + for key, value := range envMap { + if !currentEnv[key] || overload { + os.Setenv(key, value) + } + } + + return nil +} + +func readFile(filename string) (envMap map[string]string, err error) { + file, err := os.Open(filename) + if err != nil { + return + } + defer file.Close() + return ReadFromReader(file) +} + +func readString(content string) (envMap map[string]string, err error) { + return ReadFromReader(strings.NewReader(content)) +} + +func ReadFromReader(r io.Reader) (envMap map[string]string, err error) { + envMap = make(map[string]string) + + var lines []string + scanner := bufio.NewScanner(r) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + if err = scanner.Err(); err != nil { + return + } + + for _, fullLine := range lines { + if !isIgnoredLine(fullLine) { + var key, value string + key, value, err = parseLine(fullLine) + + if err != nil { + return + } + envMap[key] = value + } + } + return +} + +func parseLine(line string) (key string, value string, err error) { + if len(line) == 0 { + err = errors.New("zero length string") + return + } + + // ditch the comments (but keep quoted hashes) + if strings.Contains(line, "#") { + segmentsBetweenHashes := strings.Split(line, "#") + quotesAreOpen := false + var segmentsToKeep []string + for _, segment := range segmentsBetweenHashes { + if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 { + if quotesAreOpen { + quotesAreOpen = false + segmentsToKeep = append(segmentsToKeep, segment) + } else { + quotesAreOpen = true + } + } + + if len(segmentsToKeep) == 0 || quotesAreOpen { + segmentsToKeep = append(segmentsToKeep, segment) + } + } + + line = strings.Join(segmentsToKeep, "#") + } + + firstEquals := strings.Index(line, "=") + firstColon := strings.Index(line, ":") + splitString := strings.SplitN(line, "=", 2) + if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) { + //this is a yaml-style line + splitString = strings.SplitN(line, ":", 2) + } + + if len(splitString) != 2 { + err = errors.New("Can't separate key from value for line: " + line) + return + } + + // Parse the key + key = splitString[0] + if strings.HasPrefix(key, "export") { + key = strings.TrimPrefix(key, "export") + } + key = strings.Trim(key, " ") + + // Parse the value + value = parseValue(splitString[1]) + return +} + +func parseValue(value string) string { + + // trim + value = strings.Trim(value, " ") + + // check if we've got quoted values + if value != "" { + first := string(value[0:1]) + last := string(value[len(value)-1:]) + if first == last && strings.ContainsAny(first, `"'`) { + // pull the quotes off the edges + value = value[1 : len(value)-1] + // handle escapes + escapeRegex := regexp.MustCompile(`\\.`) + value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string { + c := strings.TrimPrefix(match, `\`) + switch c { + case "n": + return "\n" + case "r": + return "\r" + default: + return c + } + }) + } + } + + return value +} + +func isIgnoredLine(line string) bool { + trimmedLine := strings.Trim(line, " \n\t") + return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#") +} + +func doubleQuoteEscape(line string) string { + line = strings.Replace(line, `\`, `\\`, -1) + line = strings.Replace(line, "\n", `\n`, -1) + line = strings.Replace(line, "\r", `\r`, -1) + line = strings.Replace(line, `"`, `\"`, -1) + return line +} diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go new file mode 100644 index 000000000..ee4e5eb3f --- /dev/null +++ b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go @@ -0,0 +1,352 @@ +package godotenv + +import ( + "fmt" + "os" + "reflect" + "testing" +) + +var noopPresets = make(map[string]string) + +func parseAndCompare(t *testing.T, rawEnvLine string, expectedKey string, expectedValue string) { + key, value, _ := parseLine(rawEnvLine) + if key != expectedKey || value != expectedValue { + t.Errorf("Expected '%v' to parse as '%v' => '%v', got '%v' => '%v' instead", rawEnvLine, expectedKey, expectedValue, key, value) + } +} + +func loadEnvAndCompareValues(t *testing.T, loader func(files ...string) error, envFileName string, expectedValues map[string]string, presets map[string]string) { + // first up, clear the env + os.Clearenv() + + for k, v := range presets { + os.Setenv(k, v) + } + + err := loader(envFileName) + if err != nil { + t.Fatalf("Error loading %v", envFileName) + } + + for k := range expectedValues { + envValue := os.Getenv(k) + v := expectedValues[k] + if envValue != v { + t.Errorf("Mismatch for key '%v': expected '%v' got '%v'", k, v, envValue) + } + } +} + +func TestLoadWithNoArgsLoadsDotEnv(t *testing.T) { + err := Load() + pathError := err.(*os.PathError) + if pathError == nil || pathError.Op != "open" || pathError.Path != ".env" { + t.Errorf("Didn't try and open .env by default") + } +} + +func TestOverloadWithNoArgsOverloadsDotEnv(t *testing.T) { + err := Overload() + pathError := err.(*os.PathError) + if pathError == nil || pathError.Op != "open" || pathError.Path != ".env" { + t.Errorf("Didn't try and open .env by default") + } +} + +func TestLoadFileNotFound(t *testing.T) { + err := Load("somefilethatwillneverexistever.env") + if err == nil { + t.Error("File wasn't found but Load didn't return an error") + } +} + +func TestOverloadFileNotFound(t *testing.T) { + err := Overload("somefilethatwillneverexistever.env") + if err == nil { + t.Error("File wasn't found but Overload didn't return an error") + } +} + +func TestReadPlainEnv(t *testing.T) { + envFileName := "fixtures/plain.env" + expectedValues := map[string]string{ + "OPTION_A": "1", + "OPTION_B": "2", + "OPTION_C": "3", + "OPTION_D": "4", + "OPTION_E": "5", + "OPTION_F": "", + "OPTION_G": "", + } + + envMap, err := Read(envFileName) + if err != nil { + t.Error("Error reading file") + } + + if len(envMap) != len(expectedValues) { + t.Error("Didn't get the right size map back") + } + + for key, value := range expectedValues { + if envMap[key] != value { + t.Error("Read got one of the keys wrong") + } + } +} + +func TestLoadDoesNotOverride(t *testing.T) { + envFileName := "fixtures/plain.env" + + // ensure NO overload + presets := map[string]string{ + "OPTION_A": "do_not_override", + "OPTION_B": "", + } + + expectedValues := map[string]string{ + "OPTION_A": "do_not_override", + "OPTION_B": "", + } + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, presets) +} + +func TestOveroadDoesOverride(t *testing.T) { + envFileName := "fixtures/plain.env" + + // ensure NO overload + presets := map[string]string{ + "OPTION_A": "do_not_override", + } + + expectedValues := map[string]string{ + "OPTION_A": "1", + } + loadEnvAndCompareValues(t, Overload, envFileName, expectedValues, presets) +} + +func TestLoadPlainEnv(t *testing.T) { + envFileName := "fixtures/plain.env" + expectedValues := map[string]string{ + "OPTION_A": "1", + "OPTION_B": "2", + "OPTION_C": "3", + "OPTION_D": "4", + "OPTION_E": "5", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestLoadExportedEnv(t *testing.T) { + envFileName := "fixtures/exported.env" + expectedValues := map[string]string{ + "OPTION_A": "2", + "OPTION_B": "\n", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestLoadEqualsEnv(t *testing.T) { + envFileName := "fixtures/equals.env" + expectedValues := map[string]string{ + "OPTION_A": "postgres://localhost:5432/database?sslmode=disable", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestLoadQuotedEnv(t *testing.T) { + envFileName := "fixtures/quoted.env" + expectedValues := map[string]string{ + "OPTION_A": "1", + "OPTION_B": "2", + "OPTION_C": "", + "OPTION_D": "\n", + "OPTION_E": "1", + "OPTION_F": "2", + "OPTION_G": "", + "OPTION_H": "\n", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestActualEnvVarsAreLeftAlone(t *testing.T) { + os.Clearenv() + os.Setenv("OPTION_A", "actualenv") + _ = Load("fixtures/plain.env") + + if os.Getenv("OPTION_A") != "actualenv" { + t.Error("An ENV var set earlier was overwritten") + } +} + +func TestParsing(t *testing.T) { + // unquoted values + parseAndCompare(t, "FOO=bar", "FOO", "bar") + + // parses values with spaces around equal sign + parseAndCompare(t, "FOO =bar", "FOO", "bar") + parseAndCompare(t, "FOO= bar", "FOO", "bar") + + // parses double quoted values + parseAndCompare(t, `FOO="bar"`, "FOO", "bar") + + // parses single quoted values + parseAndCompare(t, "FOO='bar'", "FOO", "bar") + + // parses escaped double quotes + parseAndCompare(t, `FOO="escaped\"bar"`, "FOO", `escaped"bar`) + + // parses single quotes inside double quotes + parseAndCompare(t, `FOO="'d'"`, "FOO", `'d'`) + + // parses yaml style options + parseAndCompare(t, "OPTION_A: 1", "OPTION_A", "1") + + //parses yaml values with equal signs + parseAndCompare(t, "OPTION_A: Foo=bar", "OPTION_A", "Foo=bar") + + // parses non-yaml options with colons + parseAndCompare(t, "OPTION_A=1:B", "OPTION_A", "1:B") + + // parses export keyword + parseAndCompare(t, "export OPTION_A=2", "OPTION_A", "2") + parseAndCompare(t, `export OPTION_B='\n'`, "OPTION_B", "\n") + + // it 'expands newlines in quoted strings' do + // expect(env('FOO="bar\nbaz"')).to eql('FOO' => "bar\nbaz") + parseAndCompare(t, `FOO="bar\nbaz"`, "FOO", "bar\nbaz") + + // it 'parses varibales with "." in the name' do + // expect(env('FOO.BAR=foobar')).to eql('FOO.BAR' => 'foobar') + parseAndCompare(t, "FOO.BAR=foobar", "FOO.BAR", "foobar") + + // it 'parses varibales with several "=" in the value' do + // expect(env('FOO=foobar=')).to eql('FOO' => 'foobar=') + parseAndCompare(t, "FOO=foobar=", "FOO", "foobar=") + + // it 'strips unquoted values' do + // expect(env('foo=bar ')).to eql('foo' => 'bar') # not 'bar ' + parseAndCompare(t, "FOO=bar ", "FOO", "bar") + + // it 'ignores inline comments' do + // expect(env("foo=bar # this is foo")).to eql('foo' => 'bar') + parseAndCompare(t, "FOO=bar # this is foo", "FOO", "bar") + + // it 'allows # in quoted value' do + // expect(env('foo="bar#baz" # comment')).to eql('foo' => 'bar#baz') + parseAndCompare(t, `FOO="bar#baz" # comment`, "FOO", "bar#baz") + parseAndCompare(t, "FOO='bar#baz' # comment", "FOO", "bar#baz") + parseAndCompare(t, `FOO="bar#baz#bang" # comment`, "FOO", "bar#baz#bang") + + // it 'parses # in quoted values' do + // expect(env('foo="ba#r"')).to eql('foo' => 'ba#r') + // expect(env("foo='ba#r'")).to eql('foo' => 'ba#r') + parseAndCompare(t, `FOO="ba#r"`, "FOO", "ba#r") + parseAndCompare(t, "FOO='ba#r'", "FOO", "ba#r") + + //newlines and backslashes should be escaped + parseAndCompare(t, `FOO="bar\n\ b\az"`, "FOO", "bar\n baz") + parseAndCompare(t, `FOO="bar\\\n\ b\az"`, "FOO", "bar\\\n baz") + parseAndCompare(t, `FOO="bar\\r\ b\az"`, "FOO", "bar\\r baz") + + // it 'throws an error if line format is incorrect' do + // expect{env('lol$wut')}.to raise_error(Dotenv::FormatError) + badlyFormattedLine := "lol$wut" + _, _, err := parseLine(badlyFormattedLine) + if err == nil { + t.Errorf("Expected \"%v\" to return error, but it didn't", badlyFormattedLine) + } +} + +func TestLinesToIgnore(t *testing.T) { + // it 'ignores empty lines' do + // expect(env("\n \t \nfoo=bar\n \nfizz=buzz")).to eql('foo' => 'bar', 'fizz' => 'buzz') + if !isIgnoredLine("\n") { + t.Error("Line with nothing but line break wasn't ignored") + } + + if !isIgnoredLine("\t\t ") { + t.Error("Line full of whitespace wasn't ignored") + } + + // it 'ignores comment lines' do + // expect(env("\n\n\n # HERE GOES FOO \nfoo=bar")).to eql('foo' => 'bar') + if !isIgnoredLine("# comment") { + t.Error("Comment wasn't ignored") + } + + if !isIgnoredLine("\t#comment") { + t.Error("Indented comment wasn't ignored") + } + + // make sure we're not getting false positives + if isIgnoredLine(`export OPTION_B='\n'`) { + t.Error("ignoring a perfectly valid line to parse") + } +} + +func TestErrorReadDirectory(t *testing.T) { + envFileName := "fixtures/" + envMap, err := Read(envFileName) + + if err == nil { + t.Errorf("Expected error, got %v", envMap) + } +} + +func TestErrorParsing(t *testing.T) { + envFileName := "fixtures/invalid1.env" + envMap, err := Read(envFileName) + if err == nil { + t.Errorf("Expected error, got %v", envMap) + } +} + +func TestWrite(t *testing.T) { + writeAndCompare := func(env string, expected string) { + envMap, _ := readString(env) + actual, _ := WriteString(envMap) + if expected != actual { + t.Errorf("Expected '%v' (%v) to write as '%v', got '%v' instead.", env, envMap, expected, actual) + } + } + //just test some single lines to show the general idea + //TestRoundtrip makes most of the good assertions + + //values are always double-quoted + writeAndCompare(`key=value`, `key="value"`) + //double-quotes are escaped + writeAndCompare(`key=va"lu"e`, `key="va\"lu\"e"`) + //but single quotes are left alone + writeAndCompare(`key=va'lu'e`, `key="va'lu'e"`) + // newlines and backslashes are escaped + writeAndCompare(`foo="ba\n\r\\r!"`, `foo="ba\n\r\\r!"`) +} + +func TestRoundtrip(t *testing.T) { + fixtures := []string{"equals.env", "exported.env", "invalid1.env", "plain.env", "quoted.env"} + for _, fixture := range fixtures { + fixtureFilename := fmt.Sprintf("fixtures/%s", fixture) + env, err := readFile(fixtureFilename) + if err != nil { + continue + } + rep, err := WriteString(env) + if err != nil { + continue + } + roundtripped, err := readString(rep) + if err != nil { + continue + } + if !reflect.DeepEqual(env, roundtripped) { + t.Errorf("Expected '%s' to roundtrip as '%v', got '%v' instead", fixtureFilename, env, roundtripped) + } + + } +} From 32414b343a11604d364d7b596cc0b4447de86dd5 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 21 May 2017 16:13:53 -0400 Subject: [PATCH 22/51] [config] show empty config when ENV file doesn't exist yet --- plugins/config/src/configenv/configenv.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index c1abb0341..7300f8e7d 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -113,13 +113,6 @@ func LoadApp(appName string) (*Env, error) { if err != nil { return nil, err } - - if _, err := os.Stat(appfile); os.IsNotExist(err) { - _, err := os.Create(appfile) - if err != nil { - common.LogFail(err.Error()) - } - } return loadFromFile(appName, appfile) } @@ -138,14 +131,19 @@ func NewFromString(rep string) (*Env, error) { } return env, err } -func loadFromFile(name string, filename string) (*Env, error) { - envMap, err := godotenv.Read(filename) - env := &Env{ +func loadFromFile(name string, filename string) (env *Env, err error) { + envMap := make(map[string]string) + + if _, err := os.Stat(filename); err == nil { + envMap, err = godotenv.Read(filename) + } + + env = &Env{ name, filename, envMap, } - return env, err + return } //Merge merges the given environment on top of the reciever From e7a81a8f8b54d1f7fdfde7e8abc9cb23b4b3ce57 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 21 May 2017 16:38:55 -0400 Subject: [PATCH 23/51] [config] error message when no key is given for config:get --- plugins/config/src/subcommands/get/get.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/config/src/subcommands/get/get.go b/plugins/config/src/subcommands/get/get.go index 1c6194edc..f9d4fa162 100644 --- a/plugins/config/src/subcommands/get/get.go +++ b/plugins/config/src/subcommands/get/get.go @@ -21,6 +21,9 @@ func main() { if len(keys) > 1 { common.LogFail(fmt.Sprintf("Unexpected argument(s): %v", keys[1:])) } + if len(keys) == 0 { + common.LogFail("Expected: key") + } value := config.GetWithDefault(appName, keys[0], "") if *quoted { From 4f5c747c9961bee9bb82e1b5a46a6ec16b02c609 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 16 Jul 2017 19:50:50 -0400 Subject: [PATCH 24/51] [config] add config:export --format shell --- plugins/config/src/configenv/configenv.go | 15 +++++++++++++++ plugins/config/src/configenv/configenv_test.go | 1 + plugins/config/src/subcommands/export/export.go | 8 ++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index 7300f8e7d..5feef46b7 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -16,12 +16,18 @@ import ( common "github.com/dokku/dokku/plugins/common" ) +//ExportFormat types of possible exports type ExportFormat int const ( + //Exports format: Sourceable exports Exports ExportFormat = iota + //Envfile format: dotenv file Envfile + //DockerArgs format: --env args for docker DockerArgs + //Shell format: env arguments for shell + Shell ) //Env is a representation for global or app environment @@ -35,6 +41,7 @@ func (e *Env) String() string { return e.EnvfileString() } +//Export the Env in the given format func (e *Env) Export(format ExportFormat) string { switch format { case Exports: @@ -43,6 +50,8 @@ func (e *Env) Export(format ExportFormat) string { return e.EnvfileString() case DockerArgs: return e.DockerArgsString() + case Shell: + return e.ShellString() default: common.LogFail(fmt.Sprintf("Unknown export format: %v", format)) return "" @@ -65,6 +74,12 @@ func (e *Env) DockerArgsString() string { return e.stringWithPrefixAndSeparator("--env=", " ", true) } +//ShellString gets the contents of this Env in the form "KEY='value' KEY2='value'" +// for passing the environment in the shell +func (e *Env) ShellString() string { + return e.stringWithPrefixAndSeparator("", " ", true) +} + //StringWithPrefixAndSeparator makes a string of the environment // with the given prefix and separator for each entry func (e *Env) stringWithPrefixAndSeparator(prefix string, separator string, allowNewlines bool) string { diff --git a/plugins/config/src/configenv/configenv_test.go b/plugins/config/src/configenv/configenv_test.go index 9b86ae452..9d423e95e 100644 --- a/plugins/config/src/configenv/configenv_test.go +++ b/plugins/config/src/configenv/configenv_test.go @@ -50,6 +50,7 @@ func TestArrayExport(t *testing.T) { e, _ := NewFromString("BAR='BAZ'\nFOO='b'ar '") Expect(e.EnvfileString()).To(Equal("BAR=\"BAZ\"\nFOO=\"b'ar \"")) Expect(e.DockerArgsString()).To(Equal("--env=BAR='BAZ' --env=FOO='b'\\''ar '")) + Expect(e.ShellString()).To(Equal("BAR='BAZ' FOO='b'\\''ar '")) } func pairs(vars ...string) map[string]string { diff --git a/plugins/config/src/subcommands/export/export.go b/plugins/config/src/subcommands/export/export.go index 24840ebf3..0c9027fbc 100644 --- a/plugins/config/src/subcommands/export/export.go +++ b/plugins/config/src/subcommands/export/export.go @@ -17,7 +17,7 @@ func main() { args := flag.NewFlagSet("config:export", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") merged := args.Bool("merged", false, "--merged: merge app environment and global environment") - format := args.String("format", "exports", "--format: [ exports | envfile | docker-args ] which format to export as)") + format := args.String("format", "exports", "--format: [ exports | envfile | docker-args | shell ] which format to export as)") args.Parse(os.Args[2:]) appName, trailingArgs := config.GetCommonArgs(*global, args.Args()) @@ -27,6 +27,7 @@ func main() { env := config.GetConfig(appName, *merged) exportType := configenv.Exports + suffix := "\n" switch *format { case "exports": exportType = configenv.Exports @@ -34,9 +35,12 @@ func main() { exportType = configenv.Envfile case "docker-args": exportType = configenv.DockerArgs + case "shell": + exportType = configenv.Shell + suffix = " " default: common.LogFail(fmt.Sprintf("Unknown export format: %v", *format)) } exported := env.Export(exportType) - fmt.Println(exported) + fmt.Print(exported + suffix) } From a22594b891369874e7571608094273b54d0b23d2 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 16 Jul 2017 19:51:27 -0400 Subject: [PATCH 25/51] [config] update docs --- docs/configuration/environment-variables.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 3b1966248..fd39408d4 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -23,10 +23,15 @@ You can set multiple environment variables at once: dokku config:set node-js-app ENV=prod COMPILE_ASSETS=1 ``` -When setting variables with whitespaces, you need to escape them: +> Note: Whitespace and special characters get tricky. If you are using dokku locally you don't need to do any special escaping. If you are using dokku over ssh you will need to backslash-escape spaces: +```shell +dokku config:set node-js-app KEY="VAL\ WITH\ SPACES" +``` + +Dokku can also read base64 encoded values. That's the easiest way to set a value with newlines or spaces. To set a value with newlines you need to base64 encode it first and pass the `--encoded` flag: ```shell -dokku config:set node-js-app KEY=\"VAL\ WITH\ SPACES\" +dokku config:set --encoded node-js-app KEY="$(base64 ~/.ssh/id_rsa)" ``` When setting or unsetting environment variables, you may wish to avoid an application restart. This is useful when developing plugins or when setting multiple environment variables in a scripted manner. To do so, use the `--no-restart` flag: @@ -35,10 +40,10 @@ When setting or unsetting environment variables, you may wish to avoid an applic dokku config:set --no-restart node-js-app ENV=prod ``` -If you wish to have the variables output in an `eval`-compatible form, you can use the `--export` flag: +If you wish to have the variables output in an `eval`-compatible form, you can use the `config:export` command ```shell -dokku config node-js-app --export +dokku config:export node-js-app # outputs variables in the form: # # export ENV='prod' @@ -51,7 +56,7 @@ eval $(dokku config node-js-app --export) You can also output the variables in a single-line for usage in command-line utilities with the `--shell` flag: ```shell -dokku config node-js-app --shell +dokku config:export node-js-app --format shell # outputs variables in the form: # From fc1129c7288fb3f2b7691fecb651fc5b60c7f8ba Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 16 Jul 2017 23:59:35 -0600 Subject: [PATCH 26/51] fix: correct flake8 issues --- tests/apps/python-flask/hello.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/apps/python-flask/hello.py b/tests/apps/python-flask/hello.py index b6a5ba771..dfbc82c94 100644 --- a/tests/apps/python-flask/hello.py +++ b/tests/apps/python-flask/hello.py @@ -7,6 +7,7 @@ app = Flask(__name__) def hello(): return 'python/flask' + @app.route('/') def get_enc(env): val = os.environ.get(env, None) @@ -14,5 +15,3 @@ def get_enc(env): return val else: abort(404) - - From a137ed64dcbeef6ed143f57970191c8ab85c419d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 17 Jul 2017 00:01:38 -0600 Subject: [PATCH 27/51] fix: Declare and assign separately to avoid masking return values. --- plugins/config/docker-args-deploy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/config/docker-args-deploy b/plugins/config/docker-args-deploy index df8dee222..367a0f80d 100755 --- a/plugins/config/docker-args-deploy +++ b/plugins/config/docker-args-deploy @@ -5,12 +5,16 @@ source "$PLUGIN_AVAILABLE_PATH/config/functions" config_docker_args() { declare desc="config docker-args plugin trigger" - local trigger="$0 config_docker_args" - local STDIN=$(cat); local APP="$1"; local IMAGE_TAG="$2"; local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") + declare APP="$1" IMAGE_TAG="$2" + local ENV_ARGS IMAGE STDIN trigger + + IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") + STDIN=$(cat); + trigger="$0 config_docker_args" verify_app_name "$APP" if ! is_image_herokuish_based "$IMAGE"; then - local ENV_ARGS="$(config_export app "$APP" --format docker-args --merged)" + ENV_ARGS="$(config_export app "$APP" --format docker-args --merged)" echo -n "$STDIN $ENV_ARGS" else echo -n "$STDIN" From be5e81b24673dbc834a412354331a04e6ced2ae5 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Tue, 18 Jul 2017 00:06:49 -0400 Subject: [PATCH 28/51] [config] config:get exit with status 1 if key is not set in env --- plugins/config/config.go | 14 ++++++++++++++ plugins/config/src/subcommands/get/get.go | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/plugins/config/config.go b/plugins/config/config.go index a39cf1c56..17785b337 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -17,6 +17,20 @@ func GetWithDefault(appName string, key string, defaultValue string) string { return env.GetDefault(key, defaultValue) } +//HasKey determines if the config given by appName has a value for the given key +func HasKey(appName string, key string) bool { + env, err := loadConfig(appName) + if err != nil { + return false + } + for _, v := range env.Keys() { + if v == key { + return true + } + } + return false +} + //SetMany variables in the environment. If appName is empty the global config is used. If restart is true the app is restarted. func SetMany(appName string, entries map[string]string, restart bool) { global := appName == "" diff --git a/plugins/config/src/subcommands/get/get.go b/plugins/config/src/subcommands/get/get.go index f9d4fa162..6419bb8c7 100644 --- a/plugins/config/src/subcommands/get/get.go +++ b/plugins/config/src/subcommands/get/get.go @@ -24,8 +24,11 @@ func main() { if len(keys) == 0 { common.LogFail("Expected: key") } - value := config.GetWithDefault(appName, keys[0], "") + if !config.HasKey(appName, keys[0]) { + os.Exit(1) + } + value := config.GetWithDefault(appName, keys[0], "") if *quoted { fmt.Printf("'%s'", configenv.SingleQuoteEscape(value)) } else { From 4808c047ee3b8291aa9a0de6aab2a0f53f689819 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Wed, 19 Jul 2017 09:23:06 -0400 Subject: [PATCH 29/51] [config] make HasKey more idiomatic --- plugins/config/config.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/config/config.go b/plugins/config/config.go index 17785b337..8aed5174e 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -23,12 +23,8 @@ func HasKey(appName string, key string) bool { if err != nil { return false } - for _, v := range env.Keys() { - if v == key { - return true - } - } - return false + _, ok := env.Get(key) + return ok } //SetMany variables in the environment. If appName is empty the global config is used. If restart is true the app is restarted. From 8e8b6d60f093fea74a348aa4070349eb352c6fa2 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Wed, 19 Jul 2017 09:45:58 -0400 Subject: [PATCH 30/51] fix: Declare and assign separately to avoid masking return values. --- plugins/00_dokku-standard/exec-app-json-scripts | 8 +++++--- plugins/build-env/pre-build-buildpack | 9 ++++++--- plugins/common/functions | 9 +++++---- plugins/config/glide.yaml | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/plugins/00_dokku-standard/exec-app-json-scripts b/plugins/00_dokku-standard/exec-app-json-scripts index 8077e7a65..172dfdf99 100755 --- a/plugins/00_dokku-standard/exec-app-json-scripts +++ b/plugins/00_dokku-standard/exec-app-json-scripts @@ -23,8 +23,10 @@ get_phase_script() { execute_script() { declare desc="executes appropriate phase script key from app.json" - local APP="$1"; local IMAGE_TAG="$2"; local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") - local PHASE_SCRIPT_KEY="$3" + local APP="$1"; local IMAGE_TAG="$2"; local PHASE_SCRIPT_KEY="$3" + local IMAGE id + + IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") local SCRIPT_CMD=$(get_phase_script "$IMAGE" "$PHASE_SCRIPT_KEY" 2>/dev/null) if [[ -n "$SCRIPT_CMD" ]];then dokku_log_info1 "Running '$SCRIPT_CMD' in app container" @@ -59,7 +61,7 @@ execute_script() { declare -a ARG_ARRAY eval "ARG_ARRAY=($DOCKER_ARGS)" - local id=$(docker run "$DOKKU_GLOBAL_RUN_ARGS" -e DOKKU_TRACE="$DOKKU_TRACE" --label=dokku_phase_script="${PHASE_SCRIPT_KEY}" -d -v "$CACHE_DIR:/cache" "${ARG_ARRAY[@]}" "$IMAGE" /bin/bash -c "$COMMAND") + id=$(docker run "$DOKKU_GLOBAL_RUN_ARGS" -e DOKKU_TRACE="$DOKKU_TRACE" --label=dokku_phase_script="${PHASE_SCRIPT_KEY}" -d -v "$CACHE_DIR:/cache" "${ARG_ARRAY[@]}" "$IMAGE" /bin/bash -c "$COMMAND") if test "$(docker wait "$id")" -eq 0; then dokku_container_log_verbose_quiet "$id" if [[ "$PHASE_SCRIPT_KEY" != "postdeploy" ]];then diff --git a/plugins/build-env/pre-build-buildpack b/plugins/build-env/pre-build-buildpack index 4a6e980c1..c874eca53 100755 --- a/plugins/build-env/pre-build-buildpack +++ b/plugins/build-env/pre-build-buildpack @@ -6,8 +6,11 @@ source "$PLUGIN_AVAILABLE_PATH/config/functions" build_env_pre_build_buildpack() { declare desc="build-env pre-build-buildpack plugin trigger" local trigger="build_env_pre_build_buildpack" - local APP="$1"; local IMAGE=$(get_app_image_name "$APP"); + local APP="$1" + local IMAGE id + verify_app_name "$APP" + IMAGE=$(get_app_image_name "$APP"); [[ -z $(config_get --global CURL_CONNECT_TIMEOUT) ]] && config_set --global CURL_CONNECT_TIMEOUT=90 [[ -z $(config_get --global CURL_TIMEOUT) ]] && config_set --global CURL_TIMEOUT=60 @@ -15,12 +18,12 @@ build_env_pre_build_buildpack() { dokku_log_info1 "Adding BUILD_ENV to build environment..." # create build env files for use in buildpacks like this: # https://github.com/niteoweb/heroku-buildpack-buildout/blob/5879fa3418f7d8e079f1aa5816ba1adde73f4948/bin/compile#L34 - local id=$(config_bundle --merged "$APP" | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "mkdir -p /tmp/env; cat | tar -x -C /tmp/env") + id=$(config_bundle --merged "$APP" | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "mkdir -p /tmp/env; cat | tar -x -C /tmp/env") test "$(docker wait "$id")" -eq 0 docker commit "$id" "$IMAGE" > /dev/null # create build env for 'old style' buildpacks and dokku plugins - local id=$(config_export app "$APP" --format envfile --merged | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "cat >> /app/.env") + id=$(config_export app "$APP" --format envfile --merged | docker run "$DOKKU_GLOBAL_RUN_ARGS" -i -a stdin "$IMAGE" /bin/bash -c "cat >> /app/.env") test "$(docker wait "$id")" -eq 0 docker commit "$id" "$IMAGE" > /dev/null } diff --git a/plugins/common/functions b/plugins/common/functions index 1db8dba99..8be21e1a8 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -440,6 +440,7 @@ dokku_build() { source "$PLUGIN_AVAILABLE_PATH/config/functions" local APP="$1"; local IMAGE_SOURCE_TYPE="$2"; local TMP_WORK_DIR="$3"; local IMAGE=$(get_app_image_name "$APP") + local id verify_app_name "$APP" local CACHE_DIR="$DOKKU_ROOT/$APP/cache" @@ -461,7 +462,7 @@ dokku_build() { declare -a ARG_ARRAY eval "ARG_ARRAY=($DOCKER_ARGS)" # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -v $CACHE_DIR:/cache -e CACHE_PATH=/cache "${ARG_ARRAY[@]}" $IMAGE /build) + id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -v $CACHE_DIR:/cache -e CACHE_PATH=/cache "${ARG_ARRAY[@]}" $IMAGE /build) docker attach "$id" test "$(docker wait "$id")" -eq 0 docker commit "$id" "$IMAGE" > /dev/null @@ -616,7 +617,7 @@ dokku_deploy_cmd() { fi if [[ "$DOKKU_IS_APP_PROXY_ENABLED" == "true" ]]; then # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -e PORT=$port "${ARG_ARRAY[@]}" $IMAGE $START_CMD) + id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d -e PORT=$port "${ARG_ARRAY[@]}" $IMAGE $START_CMD) local ipaddr=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$id") # Docker < 1.9 compatibility if [[ -z $ipaddr ]]; then @@ -624,13 +625,13 @@ dokku_deploy_cmd() { fi else # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d $DOKKU_DOCKER_PORT_ARGS -e PORT=$port "${ARG_ARRAY[@]}" $IMAGE $START_CMD) + id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d $DOKKU_DOCKER_PORT_ARGS -e PORT=$port "${ARG_ARRAY[@]}" $IMAGE $START_CMD) local port=$(docker port "$id" "$port" | sed 's/[0-9.]*://') local ipaddr=127.0.0.1 fi else # shellcheck disable=SC2086 - local id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d "${ARG_ARRAY[@]}" $IMAGE $START_CMD) + id=$(docker run $DOKKU_GLOBAL_RUN_ARGS -d "${ARG_ARRAY[@]}" $IMAGE $START_CMD) fi kill_new() { diff --git a/plugins/config/glide.yaml b/plugins/config/glide.yaml index 5e318bb53..62e21eae6 100644 --- a/plugins/config/glide.yaml +++ b/plugins/config/glide.yaml @@ -2,4 +2,4 @@ package: . import: - package: github.com/ryanuber/columnize - package: github.com/alexquick/godotenv - version: feature-write-dotenv + version: master From a13722eec13f17b5a6552f9f9d3007ffd42a11d9 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Wed, 19 Jul 2017 09:59:44 -0400 Subject: [PATCH 31/51] [config] update glide dependencies --- plugins/config/glide.lock | 9 ++- plugins/config/glide.yaml | 7 +- plugins/config/src/configenv/configenv.go | 2 +- .../github.com/alexquick/godotenv/README.md | 8 +- .../github.com/alexquick/godotenv/godotenv.go | 35 +++++---- .../alexquick/godotenv/godotenv_test.go | 12 ++- .../github.com/ryanuber/columnize/README.md | 2 + .../ryanuber/columnize/columnize.go | 31 +++++--- .../ryanuber/columnize/columnize_test.go | 78 +++++++++++++------ 9 files changed, 128 insertions(+), 56 deletions(-) diff --git a/plugins/config/glide.lock b/plugins/config/glide.lock index dc3e9faad..025644fb0 100644 --- a/plugins/config/glide.lock +++ b/plugins/config/glide.lock @@ -1,7 +1,8 @@ -hash: 998d313aef819c8fa68d268d075079d281a7ccab0c3a16f6d8906b20c902f627 -updated: 2017-05-20T15:15:41.795778946-04:00 +hash: 84462ba8482aa8c12a928ab1851f212d558df5036311ec22469fdf86b77cbcad +updated: 2017-07-25T23:17:54.615885635-04:00 imports: - name: github.com/alexquick/godotenv - version: 147552bee9c6196b9388f15ab03fe15721264092 + version: 84c21eea73c643434a7b128dab043ffea44b80d9 - name: github.com/ryanuber/columnize - version: 0fbbb3f0e3fbdc5bae7c6cd5f6c1887ebfb76360 \ No newline at end of file + version: abc90934186a77966e2beeac62ed966aac0561d5 +testImports: [] diff --git a/plugins/config/glide.yaml b/plugins/config/glide.yaml index 62e21eae6..2365d21e1 100644 --- a/plugins/config/glide.yaml +++ b/plugins/config/glide.yaml @@ -1,5 +1,8 @@ -package: . +package: github.com/dokku/dokku/plugins/config +ignore: +- github.com/dokku/dokku/plugins/common +- github.com/onsi/gomega import: - package: github.com/ryanuber/columnize - package: github.com/alexquick/godotenv - version: master + version: ^1.1.0 diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index 5feef46b7..0b376db9d 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -138,7 +138,7 @@ func LoadGlobal() (*Env, error) { //NewFromString creates an env from the given ENVFILE contents representation func NewFromString(rep string) (*Env, error) { - envMap, err := godotenv.ReadFromReader(strings.NewReader(rep)) + envMap, err := godotenv.ReadString(rep) env := &Env{ "", "", diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/README.md b/plugins/config/vendor/github.com/alexquick/godotenv/README.md index aa6442b50..a6fb4953a 100644 --- a/plugins/config/vendor/github.com/alexquick/godotenv/README.md +++ b/plugins/config/vendor/github.com/alexquick/godotenv/README.md @@ -118,9 +118,15 @@ Contributions are most welcome! The parser itself is pretty stupidly naive and I 4. Push to the branch (`git push origin my-new-feature`) 5. Create new Pull Request +## Releases + +Releases should follow [Semver](http://semver.org/) though the first couple of releases are `v1` and `v1.1`. + +Use [annotated tags for all releases](https://github.com/joho/godotenv/issues/30). Example `git tag -a v1.2.1` + ## CI -Linux: [![wercker status](https://app.wercker.com/status/507594c2ec7e60f19403a568dfea0f78/m "wercker status")](https://app.wercker.com/project/bykey/507594c2ec7e60f19403a568dfea0f78) Windows: [![Build status](https://ci.appveyor.com/api/projects/status/9v40vnfvvgde64u4)](https://ci.appveyor.com/project/joho/godotenv) +Linux: [![Build Status](https://travis-ci.org/joho/godotenv.svg?branch=master)](https://travis-ci.org/joho/godotenv) Windows: [![Build status](https://ci.appveyor.com/api/projects/status/9v40vnfvvgde64u4)](https://ci.appveyor.com/project/joho/godotenv) ## Who? diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go index e56663427..e36dbbd9f 100644 --- a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go +++ b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go @@ -25,6 +25,8 @@ import ( "strings" ) +const doubleQuoteSpecialChars = "\\\n\r\"!$`" + // Load will read your env file(s) and load them into ENV for this process. // // Call this function as close as possible to the start of your program (ideally in main) @@ -93,6 +95,11 @@ func Read(filenames ...string) (envMap map[string]string, err error) { return } +// ReadString parses an environment file from dotenv file contents returning it as a map +func ReadString(content string) (envMap map[string]string, err error) { + return read(strings.NewReader(content)) +} + // Exec loads env vars from the specified filenames (empty map falls back to default) // then executes the cmd specified. // @@ -171,14 +178,10 @@ func readFile(filename string) (envMap map[string]string, err error) { return } defer file.Close() - return ReadFromReader(file) + return read(file) } -func readString(content string) (envMap map[string]string, err error) { - return ReadFromReader(strings.NewReader(content)) -} - -func ReadFromReader(r io.Reader) (envMap map[string]string, err error) { +func read(r io.Reader) (envMap map[string]string, err error) { envMap = make(map[string]string) var lines []string @@ -243,7 +246,7 @@ func parseLine(line string) (key string, value string, err error) { } if len(splitString) != 2 { - err = errors.New("Can't separate key from value for line: " + line) + err = errors.New("Can't separate key from value") return } @@ -264,8 +267,8 @@ func parseValue(value string) string { // trim value = strings.Trim(value, " ") - // check if we've got quoted values - if value != "" { + // check if we've got quoted values or possible escapes + if len(value) > 1 { first := string(value[0:1]) last := string(value[len(value)-1:]) if first == last && strings.ContainsAny(first, `"'`) { @@ -296,9 +299,15 @@ func isIgnoredLine(line string) bool { } func doubleQuoteEscape(line string) string { - line = strings.Replace(line, `\`, `\\`, -1) - line = strings.Replace(line, "\n", `\n`, -1) - line = strings.Replace(line, "\r", `\r`, -1) - line = strings.Replace(line, `"`, `\"`, -1) + for _, c := range doubleQuoteSpecialChars { + toReplace := "\\" + string(c) + if c == '\n' { + toReplace = `\n` + } + if c == '\r' { + toReplace = `\r` + } + line = strings.Replace(line, string(c), toReplace, -1) + } return line } diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go index ee4e5eb3f..8b353c051 100644 --- a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go +++ b/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go @@ -254,6 +254,10 @@ func TestParsing(t *testing.T) { parseAndCompare(t, `FOO="bar\\\n\ b\az"`, "FOO", "bar\\\n baz") parseAndCompare(t, `FOO="bar\\r\ b\az"`, "FOO", "bar\\r baz") + parseAndCompare(t, `="value"`, "", "value") + parseAndCompare(t, `KEY="`, "KEY", "\"") + parseAndCompare(t, `KEY="value`, "KEY", "\"value") + // it 'throws an error if line format is incorrect' do // expect{env('lol$wut')}.to raise_error(Dotenv::FormatError) badlyFormattedLine := "lol$wut" @@ -309,7 +313,7 @@ func TestErrorParsing(t *testing.T) { func TestWrite(t *testing.T) { writeAndCompare := func(env string, expected string) { - envMap, _ := readString(env) + envMap, _ := ReadString(env) actual, _ := WriteString(envMap) if expected != actual { t.Errorf("Expected '%v' (%v) to write as '%v', got '%v' instead.", env, envMap, expected, actual) @@ -324,8 +328,8 @@ func TestWrite(t *testing.T) { writeAndCompare(`key=va"lu"e`, `key="va\"lu\"e"`) //but single quotes are left alone writeAndCompare(`key=va'lu'e`, `key="va'lu'e"`) - // newlines and backslashes are escaped - writeAndCompare(`foo="ba\n\r\\r!"`, `foo="ba\n\r\\r!"`) + // newlines, backslashes, and some other special chars are escaped + writeAndCompare(`foo="$ba\n\r\\r!"`, `foo="\$ba\n\r\\r\!"`) } func TestRoundtrip(t *testing.T) { @@ -340,7 +344,7 @@ func TestRoundtrip(t *testing.T) { if err != nil { continue } - roundtripped, err := readString(rep) + roundtripped, err := ReadString(rep) if err != nil { continue } diff --git a/plugins/config/vendor/github.com/ryanuber/columnize/README.md b/plugins/config/vendor/github.com/ryanuber/columnize/README.md index e47634fc6..9a583db8d 100644 --- a/plugins/config/vendor/github.com/ryanuber/columnize/README.md +++ b/plugins/config/vendor/github.com/ryanuber/columnize/README.md @@ -56,12 +56,14 @@ config.Delim = "|" config.Glue = " " config.Prefix = "" config.Empty = "" +config.NoTrim = false ``` * `Delim` is the string by which columns of **input** are delimited * `Glue` is the string by which columns of **output** are delimited * `Prefix` is a string by which each line of **output** is prefixed * `Empty` is a string used to replace blank values found in output +* `NoTrim` is a boolean used to disable the automatic trimming of input values You can then pass the `Config` in using the `Format` method (signature below) to have text formatted to your liking. diff --git a/plugins/config/vendor/github.com/ryanuber/columnize/columnize.go b/plugins/config/vendor/github.com/ryanuber/columnize/columnize.go index 915716a10..527c1d1b3 100644 --- a/plugins/config/vendor/github.com/ryanuber/columnize/columnize.go +++ b/plugins/config/vendor/github.com/ryanuber/columnize/columnize.go @@ -18,8 +18,11 @@ type Config struct { // The string by which columns of output will be prefixed. Prefix string - // A replacement string to replace empty fields + // A replacement string to replace empty fields. Empty string + + // NoTrim disables automatic trimming of inputs. + NoTrim bool } // DefaultConfig returns a *Config with default values. @@ -29,18 +32,22 @@ func DefaultConfig() *Config { Glue: " ", Prefix: "", Empty: "", + NoTrim: false, } } // MergeConfig merges two config objects together and returns the resulting // configuration. Values from the right take precedence over the left side. func MergeConfig(a, b *Config) *Config { - var result Config = *a - // Return quickly if either side was nil - if a == nil || b == nil { - return &result + if a == nil { + return b } + if b == nil { + return a + } + + var result Config = *a if b.Delim != "" { result.Delim = b.Delim @@ -54,6 +61,9 @@ func MergeConfig(a, b *Config) *Config { if b.Empty != "" { result.Empty = b.Empty } + if b.NoTrim { + result.NoTrim = true + } return &result } @@ -83,10 +93,13 @@ func stringFormat(c *Config, widths []int, columns int) string { // elementsFromLine returns a list of elements, each representing a single // item which will belong to a column of output. func elementsFromLine(config *Config, line string) []interface{} { - seperated := strings.Split(line, config.Delim) - elements := make([]interface{}, len(seperated)) - for i, field := range seperated { - value := strings.TrimSpace(field) + separated := strings.Split(line, config.Delim) + elements := make([]interface{}, len(separated)) + for i, field := range separated { + value := field + if !config.NoTrim { + value = strings.TrimSpace(field) + } // Apply the empty value, if configured. if value == "" && config.Empty != "" { diff --git a/plugins/config/vendor/github.com/ryanuber/columnize/columnize_test.go b/plugins/config/vendor/github.com/ryanuber/columnize/columnize_test.go index 89dabaa38..a3194c3d0 100644 --- a/plugins/config/vendor/github.com/ryanuber/columnize/columnize_test.go +++ b/plugins/config/vendor/github.com/ryanuber/columnize/columnize_test.go @@ -2,6 +2,7 @@ package columnize import ( "fmt" + "reflect" "testing" crand "crypto/rand" @@ -155,6 +156,24 @@ func TestVariedInputSpacing(t *testing.T) { } } +func TestVariedInputSpacing_NoTrim(t *testing.T) { + input := []string{ + "Column A|Column B|Column C", + "x|y| z", + } + + config := DefaultConfig() + config.NoTrim = true + output := Format(input, config) + + expected := "Column A Column B Column C\n" + expected += "x y z" + + if output != expected { + t.Fatalf("\nexpected:\n%s\n\ngot:\n%s", expected, output) + } +} + func TestUnmatchedColumnCounts(t *testing.T) { input := []string{ "Column A | Column B | Column C", @@ -280,27 +299,42 @@ func TestEmptyConfigValues(t *testing.T) { } func TestMergeConfig(t *testing.T) { - conf1 := &Config{Delim: "a", Glue: "a", Prefix: "a", Empty: "a"} - conf2 := &Config{Delim: "b", Glue: "b", Prefix: "b", Empty: "b"} - conf3 := &Config{Delim: "c", Prefix: "c"} - - m := MergeConfig(conf1, conf2) - if m.Delim != "b" || m.Glue != "b" || m.Prefix != "b" || m.Empty != "b" { - t.Fatalf("bad: %#v", m) - } - - m = MergeConfig(conf1, conf3) - if m.Delim != "c" || m.Glue != "a" || m.Prefix != "c" || m.Empty != "a" { - t.Fatalf("bad: %#v", m) - } - - m = MergeConfig(conf1, nil) - if m.Delim != "a" || m.Glue != "a" || m.Prefix != "a" || m.Empty != "a" { - t.Fatalf("bad: %#v", m) - } - - m = MergeConfig(conf1, &Config{}) - if m.Delim != "a" || m.Glue != "a" || m.Prefix != "a" || m.Empty != "a" { - t.Fatalf("bad: %#v", m) + for _, tc := range []struct { + desc string + configA *Config + configB *Config + expect *Config + }{ + { + "merges b over a", + &Config{Delim: "a", Glue: "a", Prefix: "a", Empty: "a"}, + &Config{Delim: "b", Glue: "b", Prefix: "b", Empty: "b"}, + &Config{Delim: "b", Glue: "b", Prefix: "b", Empty: "b"}, + }, + { + "merges only non-empty config values", + &Config{Delim: "a", Glue: "a", Prefix: "a", Empty: "a"}, + &Config{Delim: "b", Prefix: "b"}, + &Config{Delim: "b", Glue: "a", Prefix: "b", Empty: "a"}, + }, + { + "takes b if a is nil", + nil, + &Config{Delim: "b", Glue: "b", Prefix: "b", Empty: "b"}, + &Config{Delim: "b", Glue: "b", Prefix: "b", Empty: "b"}, + }, + { + "takes a if b is nil", + &Config{Delim: "a", Glue: "a", Prefix: "a", Empty: "a"}, + nil, + &Config{Delim: "a", Glue: "a", Prefix: "a", Empty: "a"}, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + m := MergeConfig(tc.configA, tc.configB) + if !reflect.DeepEqual(m, tc.expect) { + t.Fatalf("\nexpect:\n%#v\n\nactual:\n%#v", tc.expect, m) + } + }) } } From 523429b47f188cc3a277d300f68bd44e9ba6473d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 22 Jul 2017 15:25:58 -0600 Subject: [PATCH 32/51] fix: ignore vendor directories --- .stickler.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.stickler.yml b/.stickler.yml index bb59453ca..0304acb21 100644 --- a/.stickler.yml +++ b/.stickler.yml @@ -7,3 +7,4 @@ linters: files: ignore: - './debian/*' + - '*/vendor/*' From a86c037a91b58218e76ca9118f82723ad7f9644a Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 23 Sep 2017 16:27:25 -0400 Subject: [PATCH 33/51] [config] upgrade to godotenv 1.2.0 --- plugins/config/glide.lock | 8 +-- plugins/config/glide.yaml | 6 +- plugins/config/src/configenv/configenv.go | 6 +- .../{alexquick => joho}/godotenv/.gitignore | 0 .../{alexquick => joho}/godotenv/.travis.yml | 0 .../{alexquick => joho}/godotenv/LICENCE | 0 .../{alexquick => joho}/godotenv/README.md | 32 ++++++++- .../godotenv/autoload/autoload.go | 0 .../godotenv/cmd/godotenv/cmd.go | 0 .../godotenv/fixtures/equals.env | 0 .../godotenv/fixtures/exported.env | 0 .../godotenv/fixtures/invalid1.env | 0 .../godotenv/fixtures/plain.env | 0 .../godotenv/fixtures/quoted.env | 0 .../{alexquick => joho}/godotenv/godotenv.go | 71 ++++++++++--------- .../godotenv/godotenv_test.go | 37 +++++++--- 16 files changed, 106 insertions(+), 54 deletions(-) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/.gitignore (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/.travis.yml (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/LICENCE (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/README.md (85%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/autoload/autoload.go (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/cmd/godotenv/cmd.go (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/fixtures/equals.env (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/fixtures/exported.env (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/fixtures/invalid1.env (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/fixtures/plain.env (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/fixtures/quoted.env (100%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/godotenv.go (92%) rename plugins/config/vendor/github.com/{alexquick => joho}/godotenv/godotenv_test.go (91%) diff --git a/plugins/config/glide.lock b/plugins/config/glide.lock index 025644fb0..eca79f070 100644 --- a/plugins/config/glide.lock +++ b/plugins/config/glide.lock @@ -1,8 +1,8 @@ -hash: 84462ba8482aa8c12a928ab1851f212d558df5036311ec22469fdf86b77cbcad -updated: 2017-07-25T23:17:54.615885635-04:00 +hash: dd48defef474bdf5c6e47281a256ca941d71fe1e637d11443fb49acf3e4b5326 +updated: 2017-09-23T16:25:15.559883959-04:00 imports: -- name: github.com/alexquick/godotenv - version: 84c21eea73c643434a7b128dab043ffea44b80d9 +- name: github.com/joho/godotenv + version: a79fa1e548e2c689c241d10173efd51e5d689d5b - name: github.com/ryanuber/columnize version: abc90934186a77966e2beeac62ed966aac0561d5 testImports: [] diff --git a/plugins/config/glide.yaml b/plugins/config/glide.yaml index 2365d21e1..bf96c156b 100644 --- a/plugins/config/glide.yaml +++ b/plugins/config/glide.yaml @@ -1,8 +1,8 @@ package: github.com/dokku/dokku/plugins/config -ignore: +ignore: - github.com/dokku/dokku/plugins/common - github.com/onsi/gomega import: - package: github.com/ryanuber/columnize -- package: github.com/alexquick/godotenv - version: ^1.1.0 +- package: github.com/joho/godotenv + version: ^1.2.0 diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index 0b376db9d..e0a8b0282 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -12,8 +12,8 @@ import ( "os" - godotenv "github.com/alexquick/godotenv" common "github.com/dokku/dokku/plugins/common" + godotenv "github.com/joho/godotenv" ) //ExportFormat types of possible exports @@ -60,7 +60,7 @@ func (e *Env) Export(format ExportFormat) string { //EnvfileString returns the contents of this Env in dotenv format func (e *Env) EnvfileString() string { - rep, _ := godotenv.WriteString(e.Map()) + rep, _ := godotenv.Marshal(e.Map()) return rep } @@ -138,7 +138,7 @@ func LoadGlobal() (*Env, error) { //NewFromString creates an env from the given ENVFILE contents representation func NewFromString(rep string) (*Env, error) { - envMap, err := godotenv.ReadString(rep) + envMap, err := godotenv.Unmarshal(rep) env := &Env{ "", "", diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/.gitignore b/plugins/config/vendor/github.com/joho/godotenv/.gitignore similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/.gitignore rename to plugins/config/vendor/github.com/joho/godotenv/.gitignore diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/.travis.yml b/plugins/config/vendor/github.com/joho/godotenv/.travis.yml similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/.travis.yml rename to plugins/config/vendor/github.com/joho/godotenv/.travis.yml diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/LICENCE b/plugins/config/vendor/github.com/joho/godotenv/LICENCE similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/LICENCE rename to plugins/config/vendor/github.com/joho/godotenv/LICENCE diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/README.md b/plugins/config/vendor/github.com/joho/godotenv/README.md similarity index 85% rename from plugins/config/vendor/github.com/alexquick/godotenv/README.md rename to plugins/config/vendor/github.com/joho/godotenv/README.md index a6fb4953a..1a20c78dc 100644 --- a/plugins/config/vendor/github.com/alexquick/godotenv/README.md +++ b/plugins/config/vendor/github.com/joho/godotenv/README.md @@ -1,4 +1,4 @@ -# GoDotEnv [![Build Status](https://travis-ci.org/joho/godotenv.svg?branch=master)](https://travis-ci.org/joho/godotenv) [![Build status](https://ci.appveyor.com/api/projects/status/9v40vnfvvgde64u4?svg=true)](https://ci.appveyor.com/project/joho/godotenv) +# GoDotEnv [![Build Status](https://travis-ci.org/joho/godotenv.svg?branch=master)](https://travis-ci.org/joho/godotenv) [![Build status](https://ci.appveyor.com/api/projects/status/9v40vnfvvgde64u4?svg=true)](https://ci.appveyor.com/project/joho/godotenv) [![Go Report Card](https://goreportcard.com/badge/github.com/joho/godotenv)](https://goreportcard.com/report/github.com/joho/godotenv) A Go (golang) port of the Ruby dotenv project (which loads env vars from a .env file) @@ -96,6 +96,20 @@ myEnv, err := godotenv.Read() s3Bucket := myEnv["S3_BUCKET"] ``` +... or from an `io.Reader` instead of a local file + +```go +reader := getRemoteFile() +myEnv, err := godotenv.Parse(reader) +``` + +... or from a `string` if you so desire + +```go +content := getRemoteFileContent() +myEnv, err := godotenv.Unmarshal(content) +``` + ### Command Mode Assuming you've installed the command as above and you've got `$GOPATH/bin` in your `$PATH` @@ -106,6 +120,22 @@ godotenv -f /some/path/to/.env some_command with some args If you don't specify `-f` it will fall back on the default of loading `.env` in `PWD` +### Writing Env Files + +Godotenv can also write a map representing the environment to a correctly-formatted and escaped file + +```go +env, err := godotenv.Unmarshal("KEY=value") +err := godotenv.Write(env, "./.env") +``` + +... or to a string + +```go +env, err := godotenv.Unmarshal("KEY=value") +content, err := godotenv.Marshal(env) +``` + ## Contributing Contributions are most welcome! The parser itself is pretty stupidly naive and I wouldn't be surprised if it breaks with edge cases. diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/autoload/autoload.go b/plugins/config/vendor/github.com/joho/godotenv/autoload/autoload.go similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/autoload/autoload.go rename to plugins/config/vendor/github.com/joho/godotenv/autoload/autoload.go diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/cmd/godotenv/cmd.go b/plugins/config/vendor/github.com/joho/godotenv/cmd/godotenv/cmd.go similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/cmd/godotenv/cmd.go rename to plugins/config/vendor/github.com/joho/godotenv/cmd/godotenv/cmd.go diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/equals.env b/plugins/config/vendor/github.com/joho/godotenv/fixtures/equals.env similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/fixtures/equals.env rename to plugins/config/vendor/github.com/joho/godotenv/fixtures/equals.env diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/exported.env b/plugins/config/vendor/github.com/joho/godotenv/fixtures/exported.env similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/fixtures/exported.env rename to plugins/config/vendor/github.com/joho/godotenv/fixtures/exported.env diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/invalid1.env b/plugins/config/vendor/github.com/joho/godotenv/fixtures/invalid1.env similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/fixtures/invalid1.env rename to plugins/config/vendor/github.com/joho/godotenv/fixtures/invalid1.env diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/plain.env b/plugins/config/vendor/github.com/joho/godotenv/fixtures/plain.env similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/fixtures/plain.env rename to plugins/config/vendor/github.com/joho/godotenv/fixtures/plain.env diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/fixtures/quoted.env b/plugins/config/vendor/github.com/joho/godotenv/fixtures/quoted.env similarity index 100% rename from plugins/config/vendor/github.com/alexquick/godotenv/fixtures/quoted.env rename to plugins/config/vendor/github.com/joho/godotenv/fixtures/quoted.env diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go b/plugins/config/vendor/github.com/joho/godotenv/godotenv.go similarity index 92% rename from plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go rename to plugins/config/vendor/github.com/joho/godotenv/godotenv.go index e36dbbd9f..48ae78c5d 100644 --- a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv.go +++ b/plugins/config/vendor/github.com/joho/godotenv/godotenv.go @@ -10,7 +10,7 @@ // // godotenv.Load() // -// and all the env vars declared in .env will be avaiable through os.Getenv("SOME_ENV_VAR") +// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR") package godotenv import ( @@ -95,9 +95,37 @@ func Read(filenames ...string) (envMap map[string]string, err error) { return } -// ReadString parses an environment file from dotenv file contents returning it as a map -func ReadString(content string) (envMap map[string]string, err error) { - return read(strings.NewReader(content)) +// Parse reads an env file from io.Reader, returning a map of keys and values. +func Parse(r io.Reader) (envMap map[string]string, err error) { + envMap = make(map[string]string) + + var lines []string + scanner := bufio.NewScanner(r) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + if err = scanner.Err(); err != nil { + return + } + + for _, fullLine := range lines { + if !isIgnoredLine(fullLine) { + var key, value string + key, value, err = parseLine(fullLine) + + if err != nil { + return + } + envMap[key] = value + } + } + return +} + +//Unmarshal reads an env file from a string, returning a map of keys and values. +func Unmarshal(str string) (envMap map[string]string, err error) { + return Parse(strings.NewReader(str)) } // Exec loads env vars from the specified filenames (empty map falls back to default) @@ -119,7 +147,7 @@ func Exec(filenames []string, cmd string, cmdArgs []string) error { // Write serializes the given environment and writes it to a file func Write(envMap map[string]string, filename string) error { - content, error := WriteString(envMap) + content, error := Marshal(envMap) if error != nil { return error } @@ -131,10 +159,9 @@ func Write(envMap map[string]string, filename string) error { return err } -// WriteString outputs the given environment as a dotenv-formatted environment file. -// +// Marshal outputs the given environment as a dotenv-formatted environment file. // Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. -func WriteString(envMap map[string]string) (string, error) { +func Marshal(envMap map[string]string) (string, error) { lines := make([]string, 0, len(envMap)) for k, v := range envMap { lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) @@ -178,34 +205,8 @@ func readFile(filename string) (envMap map[string]string, err error) { return } defer file.Close() - return read(file) -} -func read(r io.Reader) (envMap map[string]string, err error) { - envMap = make(map[string]string) - - var lines []string - scanner := bufio.NewScanner(r) - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - - if err = scanner.Err(); err != nil { - return - } - - for _, fullLine := range lines { - if !isIgnoredLine(fullLine) { - var key, value string - key, value, err = parseLine(fullLine) - - if err != nil { - return - } - envMap[key] = value - } - } - return + return Parse(file) } func parseLine(line string) (key string, value string, err error) { diff --git a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go b/plugins/config/vendor/github.com/joho/godotenv/godotenv_test.go similarity index 91% rename from plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go rename to plugins/config/vendor/github.com/joho/godotenv/godotenv_test.go index 8b353c051..91f4363e9 100644 --- a/plugins/config/vendor/github.com/alexquick/godotenv/godotenv_test.go +++ b/plugins/config/vendor/github.com/joho/godotenv/godotenv_test.go @@ -1,6 +1,7 @@ package godotenv import ( + "bytes" "fmt" "os" "reflect" @@ -96,6 +97,23 @@ func TestReadPlainEnv(t *testing.T) { } } +func TestParse(t *testing.T) { + envMap, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\""))) + expectedValues := map[string]string{ + "ONE": "1", + "TWO": "2", + "THREE": "3", + } + if err != nil { + t.Fatalf("error parsing env: %v", err) + } + for key, value := range expectedValues { + if envMap[key] != value { + t.Errorf("expected %s to be %s, got %s", key, value, envMap[key]) + } + } +} + func TestLoadDoesNotOverride(t *testing.T) { envFileName := "fixtures/plain.env" @@ -313,8 +331,8 @@ func TestErrorParsing(t *testing.T) { func TestWrite(t *testing.T) { writeAndCompare := func(env string, expected string) { - envMap, _ := ReadString(env) - actual, _ := WriteString(envMap) + envMap, _ := Unmarshal(env) + actual, _ := Marshal(envMap) if expected != actual { t.Errorf("Expected '%v' (%v) to write as '%v', got '%v' instead.", env, envMap, expected, actual) } @@ -330,23 +348,26 @@ func TestWrite(t *testing.T) { writeAndCompare(`key=va'lu'e`, `key="va'lu'e"`) // newlines, backslashes, and some other special chars are escaped writeAndCompare(`foo="$ba\n\r\\r!"`, `foo="\$ba\n\r\\r\!"`) + // lines should be sorted + writeAndCompare("foo=bar\nbaz=buzz", "baz=\"buzz\"\nfoo=\"bar\"") + } func TestRoundtrip(t *testing.T) { - fixtures := []string{"equals.env", "exported.env", "invalid1.env", "plain.env", "quoted.env"} + fixtures := []string{"equals.env", "exported.env", "plain.env", "quoted.env"} for _, fixture := range fixtures { fixtureFilename := fmt.Sprintf("fixtures/%s", fixture) env, err := readFile(fixtureFilename) if err != nil { - continue + t.Errorf("Expected '%s' to read without error (%v)", fixtureFilename, err) } - rep, err := WriteString(env) + rep, err := Marshal(env) if err != nil { - continue + t.Errorf("Expected '%s' to Marshal (%v)", fixtureFilename, err) } - roundtripped, err := ReadString(rep) + roundtripped, err := Unmarshal(rep) if err != nil { - continue + t.Errorf("Expected '%s' to Mashal and Unmarshal (%v)", fixtureFilename, err) } if !reflect.DeepEqual(env, roundtripped) { t.Errorf("Expected '%s' to roundtrip as '%v', got '%v' instead", fixtureFilename, env, roundtripped) From a8563489e7cc90ee63fdcf2a5e20942f7254c2b6 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 23 Sep 2017 17:17:59 -0400 Subject: [PATCH 34/51] [common] fix PlugnTrigger --- plugins/common/common.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index 92c9076fc..49e847e05 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -147,7 +147,12 @@ func VerifyImage(image string) bool { //PlugnTrigger fire the given plugn trigger with the given args func PlugnTrigger(triggerName string, args ...string) (string, error) { - res, err := sh.Command("plugn", "trigger", triggerName, args).Output() + shellArgs := make([]interface{}, 0, len(args)) + shellArgs = append(shellArgs, "trigger", triggerName) + for _, arg := range args { + shellArgs = append(shellArgs, arg) + } + res, err := sh.Command("plugn", shellArgs...).Output() return string(res), err } From 60e3b37d39d9aa16d426a88a3fe4bcdf21888aaa Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sat, 23 Sep 2017 17:18:58 -0400 Subject: [PATCH 35/51] [config] trigger app restart through plugn rather than exec --- plugins/config/config.go | 18 ++++++++++-------- plugins/config/src/subcommands/unset/unset.go | 2 +- plugins/ps/app-restart | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100755 plugins/ps/app-restart diff --git a/plugins/config/config.go b/plugins/config/config.go index 8aed5174e..b0ef6e435 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -48,14 +48,12 @@ func SetMany(appName string, entries map[string]string, restart bool) { } if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { - common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) - cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) - cmd.Execute() + Restart(appName) } } -//Unset a value in a config. If appName is empty the global config is used. If restart is true the app is restarted. -func Unset(appName string, keys []string, restart bool) { +//UnsetMany a value in a config. If appName is empty the global config is used. If restart is true the app is restarted. +func UnsetMany(appName string, keys []string, restart bool) { global := appName == "" env := GetConfig(appName, false) var changed = false @@ -72,9 +70,7 @@ func Unset(appName string, keys []string, restart bool) { } if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { - common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) - cmd := common.NewTokenizedShellCmd("dokku", "ps:restart", appName) - cmd.Execute() + Restart(appName) } } @@ -125,6 +121,12 @@ func GetConfig(appName string, merged bool) *configenv.Env { return env } +//Restart trigger restart on app +func Restart(appName string) { + common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) + common.PlugnTrigger("app-restart", appName) +} + func loadConfig(appName string) (*configenv.Env, error) { if appName == "" || appName == "--global" { return configenv.LoadGlobal() diff --git a/plugins/config/src/subcommands/unset/unset.go b/plugins/config/src/subcommands/unset/unset.go index 856854612..9d388a762 100644 --- a/plugins/config/src/subcommands/unset/unset.go +++ b/plugins/config/src/subcommands/unset/unset.go @@ -15,5 +15,5 @@ func main() { args.Parse(os.Args[2:]) appName, keys := config.GetCommonArgs(*global, args.Args()) - config.Unset(appName, keys, !*noRestart) + config.UnsetMany(appName, keys, !*noRestart) } diff --git a/plugins/ps/app-restart b/plugins/ps/app-restart new file mode 100755 index 000000000..614d4a424 --- /dev/null +++ b/plugins/ps/app-restart @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_AVAILABLE_PATH/ps/functions" + +ps_app_restart() { + declare desc="ps app-restart plugin trigger" + local trigger="ps_app_restart" + local APP="$1"; + + ps_restart "$APP" +} + +ps_app_restart "$@" From 5d72f145dff52fbcb45b721090fb7d8d1c06df6f Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 1 Oct 2017 22:50:01 -0400 Subject: [PATCH 36/51] [config] restore `dokku config` functionality and fix docs --- docs/configuration/environment-variables.md | 16 ++++++----- plugins/config/src/commands/commands.go | 32 +++++++++++++++++---- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index fd39408d4..3bfffe858 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -5,10 +5,11 @@ Typically an application will require some configuration to run properly. Dokku The `config` plugin provides the following commands to manage your variables: ``` -config (|--global) Display all global or app-specific config vars -config:get (|--global) KEY Display a global or app-specific config value -config:set (|--global) KEY1=VALUE1 [KEY2=VALUE2 ...] Set one or more config vars -config:unset (|--global) KEY1 [KEY2 ...] Unset one or more config vars +config (|--global) Display all global or app-specific config vars +config:get (|--global) KEY Display a global or app-specific config value +config:set (|--global) KEY1=VALUE1 [KEY2=VALUE2 ...] Set one or more config vars +config:unset (|--global) KEY1 [KEY2 ...] Unset one or more config vars +config:export [--format=FORMAT] [--merged] (|--global) Export all global or app-specific configuration ``` The variables are available both at run time and during the application build/compilation step for buildpack-based deploys. For security reasons - and as per [docker recommendations](https://github.com/docker/docker/issues/13490) - Dockerfile-based deploys have variables available *only* during runtime, as noted in [this issue](https://github.com/dokku/dokku/issues/1860). @@ -50,13 +51,14 @@ dokku config:export node-js-app # export COMPILE_ASSETS='1' # source in all the node-js-app app environment variables -eval $(dokku config node-js-app --export) +eval $(dokku config:export node-js-app) ``` -You can also output the variables in a single-line for usage in command-line utilities with the `--shell` flag: +You can control the format of the exported variables with the `--format` flag. +`--format=shell` will output the variables in a single-line for usage in command-line utilities: ```shell -dokku config:export node-js-app --format shell +dokku config:export --format shell node-js-app # outputs variables in the form: # diff --git a/plugins/config/src/commands/commands.go b/plugins/config/src/commands/commands.go index 040345e42..20590e88e 100644 --- a/plugins/config/src/commands/commands.go +++ b/plugins/config/src/commands/commands.go @@ -7,6 +7,9 @@ import ( "strconv" "strings" + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config/src/configenv" + "github.com/dokku/dokku/plugins/config" columnize "github.com/ryanuber/columnize" ) @@ -36,13 +39,32 @@ func main() { cmd := flag.Arg(0) switch cmd { case "config", "config:show": - target := flag.Arg(1) - env := config.GetConfig(target, false) - fmt.Print(config.PrettyPrintEnvEntries("", env.Map())) + args := flag.NewFlagSet("config:show", flag.ExitOnError) + global := args.Bool("global", false, "--global: use the global environment") + shell := args.Bool("shell", false, "--shell: in a single-line for usage in command-line utilities [deprecated]") + export := args.Bool("export", false, "--export: print the env as eval-compatible exports [deprecated]") + args.Parse(os.Args[2:]) + appName, _ := config.GetCommonArgs(*global, args.Args()) + env := config.GetConfig(appName, false) + + if *shell && *export { + common.LogFail("Only one of --shell and --export can be given") + } + if *shell { + fmt.Print(env.Export(configenv.Shell)) + } else if *export { + fmt.Println(env.Export(configenv.Exports)) + } else { + contextName := "global" + if appName != "" { + contextName = appName + } + common.LogInfo2(contextName + " config vars") + fmt.Println(config.PrettyPrintEnvEntries("", env.Map())) + } case "config:help": - usage() case "help": - fmt.Print(helpContent) + usage() default: dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT")) if err != nil { From 2851f2c63f67b4e8fedfc7c22e456e693fa9ccc5 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 5 Oct 2017 13:26:20 -0400 Subject: [PATCH 37/51] fix: remove duplicate logging functions --- plugins/common/common.go | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index e7a96cb0e..21689f5b6 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -359,32 +359,3 @@ func PlugnTrigger(triggerName string, args ...string) (string, error) { res, err := sh.Command("plugn", shellArgs...).Output() return string(res), err } - -// LogInfo1 is the info1 header formatter -func LogInfo1(text string) { - fmt.Fprintln(os.Stdout, fmt.Sprintf("-----> %s", text)) -} - -// LogInfo1Quiet is the info1 header formatter (with quiet option) -func LogInfo1Quiet(text string) { - if os.Getenv("DOKKU_QUIET_OUTPUT") != "" { - LogInfo1(text) - } -} - -// LogInfo2 is the info2 header formatter -func LogInfo2(text string) { - fmt.Fprintln(os.Stdout, fmt.Sprintf("=====> %s", text)) -} - -// LogInfo2Quiet is the info2 header formatter (with quiet option) -func LogInfo2Quiet(text string) { - if os.Getenv("DOKKU_QUIET_OUTPUT") != "" { - LogInfo2(text) - } -} - -// LogWarn is the warning log formatter -func LogWarn(text string) { - fmt.Fprintln(os.Stderr, fmt.Sprintf(" ! %s", text)) -} From e9b9091de534ded9d41dd7068486c7db79a5f135 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 21:35:33 -0400 Subject: [PATCH 38/51] [config] remove extra test targets --- tests.mk | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests.mk b/tests.mk index 4354d7ece..4db82d12a 100644 --- a/tests.mk +++ b/tests.mk @@ -105,16 +105,6 @@ else @$(QUIET) ./tests/ci/unit_test_runner.sh $$UNIT_TEST_BATCH endif -unit-tests-config: - @echo running config unit tests... - @$(QUIET) bats tests/unit/20_config.bats - -unit-tests-config-oddities: - @echo running config oddities unit tests... - @$(QUIET) bats tests/unit/30_config-oddities.bats - - - deploy-test-go-fail-predeploy: @echo deploying go-fail-predeploy app... cd tests && ./test_deploy ./apps/go-fail-predeploy dokku.me '' true From ecf67d3cdd26c15d0ae84c5377f6c689137916ff Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 21:40:26 -0400 Subject: [PATCH 39/51] [config] add eof newlines where needed --- tests/apps/config/check_deploy | 3 ++- tests/apps/dockerfile/check_deploy | 3 ++- tests/unit/30_config-oddities.bats | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/apps/config/check_deploy b/tests/apps/config/check_deploy index 46d7828d9..5833d3697 100755 --- a/tests/apps/config/check_deploy +++ b/tests/apps/config/check_deploy @@ -13,4 +13,5 @@ test_url () { base_url="$1" test_url / "config-test" test_url /hello "Hello - 'world'" \ No newline at end of file + 'world'" + \ No newline at end of file diff --git a/tests/apps/dockerfile/check_deploy b/tests/apps/dockerfile/check_deploy index 4fa457760..bdfcdbea9 100755 --- a/tests/apps/dockerfile/check_deploy +++ b/tests/apps/dockerfile/check_deploy @@ -12,4 +12,5 @@ test_url () { base_url="$1" test_url / "Hello World!" test_url /env/HELLO "Hello - 'world' " \ No newline at end of file + 'world' " + \ No newline at end of file diff --git a/tests/unit/30_config-oddities.bats b/tests/unit/30_config-oddities.bats index 681dbe91f..b5bad6587 100644 --- a/tests/unit/30_config-oddities.bats +++ b/tests/unit/30_config-oddities.bats @@ -44,4 +44,4 @@ teardown() { echo "status: "$status assert_output "$multiline" assert_success -} \ No newline at end of file +} From c896ca700c57f7d53bd8eb6487b270215166cbeb Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 21:42:18 -0400 Subject: [PATCH 40/51] [config] remove vestige of old test --- tests/apps/dockerfile/app.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/apps/dockerfile/app.json b/tests/apps/dockerfile/app.json index 80fd1ed17..b30e8d3b7 100644 --- a/tests/apps/dockerfile/app.json +++ b/tests/apps/dockerfile/app.json @@ -23,12 +23,8 @@ "WEB_CONCURRENCY": { "description": "The number of processes to run.", "generator": "echo 5" - }, - "FROM_APP_JSON": { - "description": "The number of processes to run.", - "value": "'foo\nbar'" } - }, + }, "image": "gliderlabs/herokuish", "addons": [ "dokku-postgres", From ad8bcecb5b7b99cc5aff5e6e3248e2a4488c6e51 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 22:38:18 -0400 Subject: [PATCH 41/51] [config] clean up PlugnTrigger and NewShellCmd --- plugins/common/common.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/plugins/common/common.go b/plugins/common/common.go index 21689f5b6..d0a39c15d 100644 --- a/plugins/common/common.go +++ b/plugins/common/common.go @@ -25,16 +25,11 @@ type ShellCmd struct { // NewShellCmd returns a new ShellCmd struct func NewShellCmd(command string) *ShellCmd { items := strings.Split(command, " ") - return NewTokenizedShellCmd(items...) -} - -// NewTokenizedShellCmd creates a new ShellCmd -func NewTokenizedShellCmd(command ...string) *ShellCmd { - cmd := command[0] - args := command[1:] + cmd := items[0] + args := items[1:] return &ShellCmd{ Command: exec.Command(cmd, args...), - CommandString: strings.Join(command, " "), + CommandString: command, Args: args, ShowOutput: true, } @@ -351,10 +346,11 @@ func VerifyImage(image string) bool { //PlugnTrigger fire the given plugn trigger with the given args func PlugnTrigger(triggerName string, args ...string) (string, error) { - shellArgs := make([]interface{}, 0, len(args)) - shellArgs = append(shellArgs, "trigger", triggerName) - for _, arg := range args { - shellArgs = append(shellArgs, arg) + shellArgs := make([]interface{}, len(args)+2) + shellArgs[0] = "trigger" + shellArgs[1] = triggerName + for i, arg := range args { + shellArgs[i+2] = arg } res, err := sh.Command("plugn", shellArgs...).Output() return string(res), err From b84635baf76c6484ed778c92e355f7308cd27baa Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 22:49:44 -0400 Subject: [PATCH 42/51] [config] remove old code --- plugins/config/config.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/plugins/config/config.go b/plugins/config/config.go index c170c7800..b0ef6e435 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -17,28 +17,6 @@ func GetWithDefault(appName string, key string, defaultValue string) string { return env.GetDefault(key, defaultValue) } -// GetWithDefault returns the value set for a given key, returning defaultValue if none found -// func GetWithDefault(appName string, key string, defaultValue string) (value string) { -// value = defaultValue - -// envFile := strings.Join([]string{common.MustGetEnv("DOKKU_ROOT"), appName, "ENV"}, "/") -// lines, err := common.FileToSlice(envFile) -// if err != nil { -// return -// } -// prefix := fmt.Sprintf("export %v=", key) -// for _, line := range lines { -// if !strings.HasPrefix(line, prefix) { -// continue -// } -// value = strings.TrimPrefix(line, prefix) -// if strings.HasPrefix(value, "'") && strings.HasSuffix(value, "'") { -// value = strings.TrimPrefix(strings.TrimSuffix(value, "'"), "'") -// } -// } -// return -// } - //HasKey determines if the config given by appName has a value for the given key func HasKey(appName string, key string) bool { env, err := loadConfig(appName) From 3422866909d5dfe177f986297e31b16a0694ec59 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 22:50:29 -0400 Subject: [PATCH 43/51] [config] remove spurious newlines --- plugins/config/config.go | 6 ------ plugins/config/src/configenv/configenv.go | 1 - .../config/vendor/github.com/joho/godotenv/godotenv_test.go | 2 -- 3 files changed, 9 deletions(-) diff --git a/plugins/config/config.go b/plugins/config/config.go index b0ef6e435..61d4ddbd1 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -31,14 +31,11 @@ func HasKey(appName string, key string) bool { func SetMany(appName string, entries map[string]string, restart bool) { global := appName == "" env := GetConfig(appName, false) - keys := make([]string, 0, len(entries)) - for k, v := range entries { env.Set(k, v) keys = append(keys, k) } - if len(entries) != 0 { common.LogInfo1("Setting config vars") fmt.Println(PrettyPrintEnvEntries(" ", entries)) @@ -46,7 +43,6 @@ func SetMany(appName string, entries map[string]string, restart bool) { args := append([]string{appName, "set"}, keys...) common.PlugnTrigger("post-config-update", args...) } - if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { Restart(appName) } @@ -62,13 +58,11 @@ func UnsetMany(appName string, keys []string, restart bool) { env.Unset(k) changed = true } - if changed { env.Write() args := append([]string{appName, "unset"}, keys...) common.PlugnTrigger("post-config-update", args...) } - if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { Restart(appName) } diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index e0a8b0282..be8b1632d 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -148,7 +148,6 @@ func NewFromString(rep string) (*Env, error) { } func loadFromFile(name string, filename string) (env *Env, err error) { envMap := make(map[string]string) - if _, err := os.Stat(filename); err == nil { envMap, err = godotenv.Read(filename) } diff --git a/plugins/config/vendor/github.com/joho/godotenv/godotenv_test.go b/plugins/config/vendor/github.com/joho/godotenv/godotenv_test.go index 91f4363e9..cd9e2f309 100644 --- a/plugins/config/vendor/github.com/joho/godotenv/godotenv_test.go +++ b/plugins/config/vendor/github.com/joho/godotenv/godotenv_test.go @@ -350,7 +350,6 @@ func TestWrite(t *testing.T) { writeAndCompare(`foo="$ba\n\r\\r!"`, `foo="\$ba\n\r\\r\!"`) // lines should be sorted writeAndCompare("foo=bar\nbaz=buzz", "baz=\"buzz\"\nfoo=\"bar\"") - } func TestRoundtrip(t *testing.T) { @@ -372,6 +371,5 @@ func TestRoundtrip(t *testing.T) { if !reflect.DeepEqual(env, roundtripped) { t.Errorf("Expected '%s' to roundtrip as '%v', got '%v' instead", fixtureFilename, env, roundtripped) } - } } From 3f497908b5abc3a0f2a567294eb2450a90604057 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 23:41:42 -0400 Subject: [PATCH 44/51] [config] use named returns where applicable --- plugins/config/config.go | 28 ++++++----------------- plugins/config/src/configenv/configenv.go | 23 ++++++++++--------- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/plugins/config/config.go b/plugins/config/config.go index 61d4ddbd1..401ff3c35 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -5,11 +5,10 @@ import ( "github.com/dokku/dokku/plugins/common" configenv "github.com/dokku/dokku/plugins/config/src/configenv" - columnize "github.com/ryanuber/columnize" ) //GetWithDefault gets a value from a config. If appName is empty the global config is used. -func GetWithDefault(appName string, key string, defaultValue string) string { +func GetWithDefault(appName string, key string, defaultValue string) (value string) { env, err := loadConfig(appName) if err != nil { return defaultValue @@ -18,12 +17,12 @@ func GetWithDefault(appName string, key string, defaultValue string) string { } //HasKey determines if the config given by appName has a value for the given key -func HasKey(appName string, key string) bool { +func HasKey(appName string, key string) (ok bool) { env, err := loadConfig(appName) if err != nil { return false } - _, ok := env.Get(key) + _, ok = env.Get(key) return ok } @@ -68,22 +67,9 @@ func UnsetMany(appName string, keys []string, restart bool) { } } -//PrettyPrintEnvEntries in columns -func PrettyPrintEnvEntries(prefix string, entries map[string]string) string { - colConfig := columnize.DefaultConfig() - colConfig.Prefix = prefix - colConfig.Delim = "\x00" - lines := make([]string, 0, len(entries)) - for k, v := range entries { - lines = append(lines, fmt.Sprintf("%s:\x00%s", k, v)) - } - return columnize.Format(lines, colConfig) -} - //GetCommonArgs extracts common positional args (appName and keys) -func GetCommonArgs(global bool, args []string) (string, []string) { +func GetCommonArgs(global bool, args []string) (appName string, keys []string) { nextArg := 0 - appName := "" if !global { if len(args) > 0 { appName = args[0] @@ -94,12 +80,12 @@ func GetCommonArgs(global bool, args []string) (string, []string) { nextArg++ } } - keys := args[nextArg:] + keys = args[nextArg:] return appName, keys } //GetConfig for the given app (global config if appName is empty). Merge with global config if merged is true. -func GetConfig(appName string, merged bool) *configenv.Env { +func GetConfig(appName string, merged bool) (env *configenv.Env) { env, err := loadConfig(appName) if err != nil { common.LogFail(err.Error()) @@ -121,7 +107,7 @@ func Restart(appName string) { common.PlugnTrigger("app-restart", appName) } -func loadConfig(appName string) (*configenv.Env, error) { +func loadConfig(appName string) (env *configenv.Env, err error) { if appName == "" || appName == "--global" { return configenv.LoadGlobal() } diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index be8b1632d..c07c4a677 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -123,10 +123,10 @@ func (e *Env) ExportBundle(dest io.Writer) error { } //LoadApp loads an environment for the given app -func LoadApp(appName string) (*Env, error) { +func LoadApp(appName string) (env *Env, err error) { appfile, err := getAppFile(appName) if err != nil { - return nil, err + return } return loadFromFile(appName, appfile) } @@ -137,15 +137,16 @@ func LoadGlobal() (*Env, error) { } //NewFromString creates an env from the given ENVFILE contents representation -func NewFromString(rep string) (*Env, error) { +func NewFromString(rep string) (env *Env, err error) { envMap, err := godotenv.Unmarshal(rep) - env := &Env{ + env = &Env{ "", "", envMap, } - return env, err + return } + func loadFromFile(name string, filename string) (env *Env, err error) { envMap := make(map[string]string) if _, err := os.Stat(filename); err == nil { @@ -178,19 +179,19 @@ func (e *Env) Unset(key string) { } //Keys gets the keys in this environment -func (e *Env) Keys() []string { - keys := make([]string, 0, len(e.env)) +func (e *Env) Keys() (keys []string) { + keys = make([]string, 0, len(e.env)) for k := range e.env { keys = append(keys, k) } sort.Strings(keys) - return keys + return } //Get an environment variable -func (e *Env) Get(key string) (string, bool) { - v, ok := e.env[key] - return v, ok +func (e *Env) Get(key string) (value string, ok bool) { + value, ok = e.env[key] + return } //GetDefault an environment variable or a default if it doesnt exist From 430cc2b4cabab54305ec247b7f7bada09e43bbd1 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 23:42:14 -0400 Subject: [PATCH 45/51] [config] move PrettyPrintEnvEntries --- plugins/config/config.go | 2 +- plugins/config/src/commands/commands.go | 2 +- plugins/config/src/configenv/configenv.go | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/plugins/config/config.go b/plugins/config/config.go index 401ff3c35..545350665 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -37,7 +37,7 @@ func SetMany(appName string, entries map[string]string, restart bool) { } if len(entries) != 0 { common.LogInfo1("Setting config vars") - fmt.Println(PrettyPrintEnvEntries(" ", entries)) + fmt.Println(configenv.PrettyPrintEnvEntries(" ", entries)) env.Write() args := append([]string{appName, "set"}, keys...) common.PlugnTrigger("post-config-update", args...) diff --git a/plugins/config/src/commands/commands.go b/plugins/config/src/commands/commands.go index 20590e88e..afbe5246f 100644 --- a/plugins/config/src/commands/commands.go +++ b/plugins/config/src/commands/commands.go @@ -60,7 +60,7 @@ func main() { contextName = appName } common.LogInfo2(contextName + " config vars") - fmt.Println(config.PrettyPrintEnvEntries("", env.Map())) + fmt.Println(env.Export(configenv.Pretty)) } case "config:help": case "help": diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index c07c4a677..623cf0e27 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -14,6 +14,7 @@ import ( common "github.com/dokku/dokku/plugins/common" godotenv "github.com/joho/godotenv" + "github.com/ryanuber/columnize" ) //ExportFormat types of possible exports @@ -28,6 +29,8 @@ const ( DockerArgs //Shell format: env arguments for shell Shell + //Pretty format: pretty-printed in columns + Pretty ) //Env is a representation for global or app environment @@ -52,6 +55,8 @@ func (e *Env) Export(format ExportFormat) string { return e.DockerArgsString() case Shell: return e.ShellString() + case Pretty: + return PrettyPrintEnvEntries("", e.Map()) default: common.LogFail(fmt.Sprintf("Unknown export format: %v", format)) return "" @@ -242,3 +247,15 @@ func getAppFile(appName string) (string, error) { func getGlobalFile() string { return filepath.Join(common.MustGetEnv("DOKKU_ROOT"), "ENV") } + +//PrettyPrintEnvEntries in columns +func PrettyPrintEnvEntries(prefix string, entries map[string]string) (representation string) { + colConfig := columnize.DefaultConfig() + colConfig.Prefix = prefix + colConfig.Delim = "\x00" + lines := make([]string, 0, len(entries)) + for k, v := range entries { + lines = append(lines, fmt.Sprintf("%s:\x00%s", k, v)) + } + return columnize.Format(lines, colConfig) +} From 03830562c28552a5233be089f796ed8cb79d3a32 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Thu, 12 Oct 2017 23:48:16 -0400 Subject: [PATCH 46/51] move around the members of configenv to make more sense --- plugins/config/src/configenv/configenv.go | 238 +++++++++++----------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/src/configenv/configenv.go index 623cf0e27..5ddd93732 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/src/configenv/configenv.go @@ -40,10 +40,105 @@ type Env struct { env map[string]string } +//NewFromString creates an env from the given ENVFILE contents representation +func NewFromString(rep string) (env *Env, err error) { + envMap, err := godotenv.Unmarshal(rep) + env = &Env{ + "", + "", + envMap, + } + return +} + +//LoadApp loads an environment for the given app +func LoadApp(appName string) (env *Env, err error) { + appfile, err := getAppFile(appName) + if err != nil { + return + } + return loadFromFile(appName, appfile) +} + +//LoadGlobal loads the global environmen +func LoadGlobal() (*Env, error) { + return loadFromFile("global", getGlobalFile()) +} + +//Get an environment variable +func (e *Env) Get(key string) (value string, ok bool) { + value, ok = e.env[key] + return +} + +//GetDefault an environment variable or a default if it doesnt exist +func (e *Env) GetDefault(key string, defaultValue string) string { + v, ok := e.env[key] + if !ok { + return defaultValue + } + return v +} + +//GetBoolDefault gets the bool value of the given key with the given default +//right now that is evaluated as `value != "0"` +func (e *Env) GetBoolDefault(key string, defaultValue bool) bool { + v, ok := e.Get(key) + if !ok { + return defaultValue + } + return v != "0" +} + +//Set an environment variable +func (e *Env) Set(key string, value string) { + e.env[key] = value +} + +//Unset an environment variable +func (e *Env) Unset(key string) { + delete(e.env, key) +} + +//Keys gets the keys in this environment +func (e *Env) Keys() (keys []string) { + keys = make([]string, 0, len(e.env)) + for k := range e.env { + keys = append(keys, k) + } + sort.Strings(keys) + return +} + +//Len return the number of items in this environment +func (e *Env) Len() int { + return len(e.env) +} + +//Map return the Env as a map +func (e *Env) Map() map[string]string { + return e.env +} + func (e *Env) String() string { return e.EnvfileString() } +//Merge merges the given environment on top of the reciever +func (e *Env) Merge(other *Env) { + for _, k := range other.Keys() { + e.Set(k, other.GetDefault(k, "")) + } +} + +//Write an Env back to the file it was read from as an exportfile +func (e *Env) Write() error { + if e.filename == "" { + return errors.New("this Env was created unbound to a file") + } + return godotenv.Write(e.Map(), e.filename) +} + //Export the Env in the given format func (e *Env) Export(format ExportFormat) string { switch format { @@ -85,26 +180,6 @@ func (e *Env) ShellString() string { return e.stringWithPrefixAndSeparator("", " ", true) } -//StringWithPrefixAndSeparator makes a string of the environment -// with the given prefix and separator for each entry -func (e *Env) stringWithPrefixAndSeparator(prefix string, separator string, allowNewlines bool) string { - keys := e.Keys() - entries := make([]string, len(keys)) - for i, k := range keys { - v := SingleQuoteEscape(e.env[k]) - if !allowNewlines { - v = strings.Replace(v, "\n", "'$'\\n''", -1) - } - entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, v) - } - return strings.Join(entries, separator) -} - -//SingleQuoteEscape escapes the value as if it were shell-quoted in single quotes -func SingleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\''aped' - return strings.Replace(value, "'", "'\\''", -1) -} - //ExportBundle writes a tarfile of the environmnet to the given io.Writer. // for every environment variable there is a file with the variable's key // with its content set to the variable's value @@ -127,29 +202,21 @@ func (e *Env) ExportBundle(dest io.Writer) error { return nil } -//LoadApp loads an environment for the given app -func LoadApp(appName string) (env *Env, err error) { - appfile, err := getAppFile(appName) - if err != nil { - return - } - return loadFromFile(appName, appfile) +//SingleQuoteEscape escapes the value as if it were shell-quoted in single quotes +func SingleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\''aped' + return strings.Replace(value, "'", "'\\''", -1) } -//LoadGlobal loads the global environmen -func LoadGlobal() (*Env, error) { - return loadFromFile("global", getGlobalFile()) -} - -//NewFromString creates an env from the given ENVFILE contents representation -func NewFromString(rep string) (env *Env, err error) { - envMap, err := godotenv.Unmarshal(rep) - env = &Env{ - "", - "", - envMap, +//PrettyPrintEnvEntries in columns +func PrettyPrintEnvEntries(prefix string, entries map[string]string) (representation string) { + colConfig := columnize.DefaultConfig() + colConfig.Prefix = prefix + colConfig.Delim = "\x00" + lines := make([]string, 0, len(entries)) + for k, v := range entries { + lines = append(lines, fmt.Sprintf("%s:\x00%s", k, v)) } - return + return columnize.Format(lines, colConfig) } func loadFromFile(name string, filename string) (env *Env, err error) { @@ -166,74 +233,19 @@ func loadFromFile(name string, filename string) (env *Env, err error) { return } -//Merge merges the given environment on top of the reciever -func (e *Env) Merge(other *Env) { - for _, k := range other.Keys() { - e.Set(k, other.GetDefault(k, "")) +//stringWithPrefixAndSeparator makes a string of the environment +// with the given prefix and separator for each entry +func (e *Env) stringWithPrefixAndSeparator(prefix string, separator string, allowNewlines bool) string { + keys := e.Keys() + entries := make([]string, len(keys)) + for i, k := range keys { + v := SingleQuoteEscape(e.env[k]) + if !allowNewlines { + v = strings.Replace(v, "\n", "'$'\\n''", -1) + } + entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, v) } -} - -//Set an environment variable -func (e *Env) Set(key string, value string) { - e.env[key] = value -} - -//Unset an environment variable -func (e *Env) Unset(key string) { - delete(e.env, key) -} - -//Keys gets the keys in this environment -func (e *Env) Keys() (keys []string) { - keys = make([]string, 0, len(e.env)) - for k := range e.env { - keys = append(keys, k) - } - sort.Strings(keys) - return -} - -//Get an environment variable -func (e *Env) Get(key string) (value string, ok bool) { - value, ok = e.env[key] - return -} - -//GetDefault an environment variable or a default if it doesnt exist -func (e *Env) GetDefault(key string, defaultValue string) string { - v, ok := e.env[key] - if !ok { - return defaultValue - } - return v -} - -//GetBoolDefault gets the bool value of the given key with the given default -//right now that is evaluated as `value != "0"` -func (e *Env) GetBoolDefault(key string, defaultValue bool) bool { - v, ok := e.Get(key) - if !ok { - return defaultValue - } - return v != "0" -} - -//Len return the number of items in this environment -func (e *Env) Len() int { - return len(e.env) -} - -//Map return the Env as a map -func (e *Env) Map() map[string]string { - return e.env -} - -//Write an Env back to the file it was read from as an exportfile -func (e *Env) Write() error { - if e.filename == "" { - return errors.New("this Env was created unbound to a file") - } - return godotenv.Write(e.Map(), e.filename) + return strings.Join(entries, separator) } func getAppFile(appName string) (string, error) { @@ -247,15 +259,3 @@ func getAppFile(appName string) (string, error) { func getGlobalFile() string { return filepath.Join(common.MustGetEnv("DOKKU_ROOT"), "ENV") } - -//PrettyPrintEnvEntries in columns -func PrettyPrintEnvEntries(prefix string, entries map[string]string) (representation string) { - colConfig := columnize.DefaultConfig() - colConfig.Prefix = prefix - colConfig.Delim = "\x00" - lines := make([]string, 0, len(entries)) - for k, v := range entries { - lines = append(lines, fmt.Sprintf("%s:\x00%s", k, v)) - } - return columnize.Format(lines, colConfig) -} From 91c2164e493a90a3708323efc08cc9bc708c9e84 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Sun, 15 Oct 2017 20:10:46 -0400 Subject: [PATCH 47/51] [config] refactor package structure --- plugins/config/config.go | 89 ++++------ plugins/config/config_test.go | 106 ++++++++++- .../configenv/configenv.go => environment.go} | 103 ++++++----- plugins/config/environment_test.go | 102 +++++++++++ plugins/config/src/commands/commands.go | 23 +-- .../config/src/configenv/configenv_test.go | 66 ------- .../config/src/subcommands/bundle/bundle.go | 10 +- .../config/src/subcommands/export/export.go | 28 +-- plugins/config/src/subcommands/get/get.go | 22 +-- plugins/config/src/subcommands/keys/keys.go | 12 +- plugins/config/src/subcommands/set/set.go | 26 +-- plugins/config/src/subcommands/unset/unset.go | 4 +- plugins/config/subcommands.go | 167 ++++++++++++++++++ 13 files changed, 469 insertions(+), 289 deletions(-) rename plugins/config/{src/configenv/configenv.go => environment.go} (74%) create mode 100644 plugins/config/environment_test.go delete mode 100644 plugins/config/src/configenv/configenv_test.go create mode 100644 plugins/config/subcommands.go diff --git a/plugins/config/config.go b/plugins/config/config.go index 545350665..575b6c3ee 100644 --- a/plugins/config/config.go +++ b/plugins/config/config.go @@ -4,32 +4,33 @@ import ( "fmt" "github.com/dokku/dokku/plugins/common" - configenv "github.com/dokku/dokku/plugins/config/src/configenv" ) -//GetWithDefault gets a value from a config. If appName is empty the global config is used. -func GetWithDefault(appName string, key string, defaultValue string) (value string) { - env, err := loadConfig(appName) +//Get retreives a value from a config. If appName is empty the global config is used. +func Get(appName string, key string) (value string, ok bool) { + env, err := loadAppOrGlobalEnv(appName) if err != nil { - return defaultValue + return "", false } - return env.GetDefault(key, defaultValue) + return env.Get(key) } -//HasKey determines if the config given by appName has a value for the given key -func HasKey(appName string, key string) (ok bool) { - env, err := loadConfig(appName) - if err != nil { - return false +//GetWithDefault gets a value from a config. If appName is empty the global config is used. If the appName or key do not exist defaultValue is returned. +func GetWithDefault(appName string, key string, defaultValue string) (value string) { + value, ok := Get(appName, key) + if !ok { + return defaultValue } - _, ok = env.Get(key) - return ok + return value } //SetMany variables in the environment. If appName is empty the global config is used. If restart is true the app is restarted. -func SetMany(appName string, entries map[string]string, restart bool) { +func SetMany(appName string, entries map[string]string, restart bool) (err error) { global := appName == "" - env := GetConfig(appName, false) + env, err := loadAppOrGlobalEnv(appName) + if err != nil { + return + } keys := make([]string, 0, len(entries)) for k, v := range entries { env.Set(k, v) @@ -37,20 +38,24 @@ func SetMany(appName string, entries map[string]string, restart bool) { } if len(entries) != 0 { common.LogInfo1("Setting config vars") - fmt.Println(configenv.PrettyPrintEnvEntries(" ", entries)) + fmt.Println(prettyPrintEnvEntries(" ", entries)) env.Write() args := append([]string{appName, "set"}, keys...) common.PlugnTrigger("post-config-update", args...) } if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { - Restart(appName) + triggerRestart(appName) } + return } //UnsetMany a value in a config. If appName is empty the global config is used. If restart is true the app is restarted. -func UnsetMany(appName string, keys []string, restart bool) { +func UnsetMany(appName string, keys []string, restart bool) (err error) { global := appName == "" - env := GetConfig(appName, false) + env, err := loadAppOrGlobalEnv(appName) + if err != nil { + return + } var changed = false for _, k := range keys { common.LogInfo1(fmt.Sprintf("Unsetting %s", k)) @@ -63,53 +68,19 @@ func UnsetMany(appName string, keys []string, restart bool) { common.PlugnTrigger("post-config-update", args...) } if !global && restart && env.GetBoolDefault("DOKKU_APP_RESTORE", true) { - Restart(appName) + triggerRestart(appName) } + return } -//GetCommonArgs extracts common positional args (appName and keys) -func GetCommonArgs(global bool, args []string) (appName string, keys []string) { - nextArg := 0 - if !global { - if len(args) > 0 { - appName = args[0] - } - if appName == "" { - common.LogFail("Please specify an app or --global") - } else { - nextArg++ - } - } - keys = args[nextArg:] - return appName, keys -} - -//GetConfig for the given app (global config if appName is empty). Merge with global config if merged is true. -func GetConfig(appName string, merged bool) (env *configenv.Env) { - env, err := loadConfig(appName) - if err != nil { - common.LogFail(err.Error()) - } - if appName != "" && merged { - global, err := configenv.LoadGlobal() - if err != nil { - common.LogFail(err.Error()) - } - global.Merge(env) - return global - } - return env -} - -//Restart trigger restart on app -func Restart(appName string) { +func triggerRestart(appName string) { common.LogInfo1(fmt.Sprintf("Restarting app %s", appName)) common.PlugnTrigger("app-restart", appName) } -func loadConfig(appName string) (env *configenv.Env, err error) { +func loadAppOrGlobalEnv(appName string) (env *Env, err error) { if appName == "" || appName == "--global" { - return configenv.LoadGlobal() + return LoadGlobalEnv() } - return configenv.LoadApp(appName) + return LoadAppEnv(appName) } diff --git a/plugins/config/config_test.go b/plugins/config/config_test.go index dbe660065..2537ed229 100644 --- a/plugins/config/config_test.go +++ b/plugins/config/config_test.go @@ -6,20 +6,29 @@ import ( "strings" "testing" + "github.com/dokku/dokku/plugins/common" + . "github.com/onsi/gomega" ) var ( - testAppName = "test-app-1" - testAppDir = strings.Join([]string{"/home/dokku/", testAppName}, "") + testAppName = "test-app-1" + dokkuRoot = common.MustGetEnv("DOKKU_ROOT") + testAppDir = strings.Join([]string{dokkuRoot, testAppName}, "/") + globalConfigFile = strings.Join([]string{dokkuRoot, "ENV"}, "/") ) func setupTestApp() (err error) { - Expect(os.MkdirAll(testAppDir, 0644)).To(Succeed()) + Expect(os.MkdirAll(testAppDir, 0766)).To(Succeed()) b := []byte("export testKey=TESTING\n") if err = ioutil.WriteFile(strings.Join([]string{testAppDir, "/ENV"}, ""), b, 0644); err != nil { return } + + b = []byte("export testKey=GLOBAL_TESTING\nexport globalKey=GLOBAL_VALUE") + if err = ioutil.WriteFile(globalConfigFile, b, 0644); err != nil { + return + } return } @@ -32,5 +41,96 @@ func TestConfigGetWithDefault(t *testing.T) { Expect(setupTestApp()).To(Succeed()) Expect(GetWithDefault(testAppName, "unknownKey", "UNKNOWN")).To(Equal("UNKNOWN")) Expect(GetWithDefault(testAppName, "testKey", "testKey")).To(Equal("TESTING")) + Expect(GetWithDefault(testAppName+"-nonexistent", "testKey", "default")).To(Equal("default")) teardownTestApp() } + +func TestConfigGet(t *testing.T) { + RegisterTestingT(t) + Expect(setupTestApp()).To(Succeed()) + defer teardownTestApp() + + expectValue(testAppName, "testKey", "TESTING") + expectValue("", "testKey", "GLOBAL_TESTING") + + expectNoValue(testAppName, "testKey2") + expectNoValue("", "testKey2") +} + +func TestConfigSetMany(t *testing.T) { + RegisterTestingT(t) + Expect(setupTestApp()).To(Succeed()) + defer teardownTestApp() + + expectValue(testAppName, "testKey", "TESTING") + + vals := map[string]string{"testKey": "updated", "testKey2": "new"} + Expect(SetMany(testAppName, vals, false)).To(Succeed()) + expectValue(testAppName, "testKey", "updated") + expectValue(testAppName, "testKey2", "new") + + vals = map[string]string{"testKey": "updated_global", "testKey2": "new_global"} + Expect(SetMany("", vals, false)).To(Succeed()) + expectValue("", "testKey", "updated_global") + expectValue("", "testKey2", "new_global") + expectValue("", "globalKey", "GLOBAL_VALUE") + expectValue(testAppName, "testKey", "updated") + expectValue(testAppName, "testKey2", "new") + + Expect(SetMany(testAppName+"does_not_exist", vals, false)).ToNot(Succeed()) +} + +func TestConfigUnsetMany(t *testing.T) { + RegisterTestingT(t) + Expect(setupTestApp()).To(Succeed()) + defer teardownTestApp() + + expectValue(testAppName, "testKey", "TESTING") + expectValue("", "testKey", "GLOBAL_TESTING") + + keys := []string{"testKey", "noKey"} + Expect(UnsetMany(testAppName, keys, false)).To(Succeed()) + expectNoValue(testAppName, "testKey") + expectValue("", "testKey", "GLOBAL_TESTING") + + Expect(UnsetMany(testAppName, keys, false)).To(Succeed()) + expectNoValue(testAppName, "testKey") + expectNoValue(testAppName, "globalKey") + + Expect(UnsetMany(testAppName+"does-not-exist", keys, false)).ToNot(Succeed()) +} + +func TestEnvironmentLoading(t *testing.T) { + RegisterTestingT(t) + Expect(setupTestApp()).To(Succeed()) + defer teardownTestApp() + + env, err := LoadMergedAppEnv(testAppName) + Expect(err).To(Succeed()) + v, _ := env.Get("testKey") + Expect(v).To(Equal("TESTING")) + v, _ = env.Get("globalKey") + Expect(v).To(Equal("GLOBAL_VALUE")) + Expect(env.Write()).ToNot(Succeed()) + + env, err = LoadAppEnv(testAppName) + env.Set("testKey", "TESTING-updated") + env.Set("testKey2", "TESTING-'\n'-updated") + env.Write() + + expectValue(testAppName, "testKey", "TESTING-updated") + expectValue(testAppName, "testKey2", "TESTING-'\n'-updated") + expectValue("", "testKey", "GLOBAL_TESTING") + Expect(err).To(Succeed()) +} + +func expectValue(appName string, key string, expected string) { + v, ok := Get(appName, key) + Expect(ok).To(Equal(true)) + Expect(v).To(Equal(expected)) +} + +func expectNoValue(appName string, key string) { + _, ok := Get(appName, key) + Expect(ok).To(Equal(false)) +} diff --git a/plugins/config/src/configenv/configenv.go b/plugins/config/environment.go similarity index 74% rename from plugins/config/src/configenv/configenv.go rename to plugins/config/environment.go index 5ddd93732..43417cd87 100644 --- a/plugins/config/src/configenv/configenv.go +++ b/plugins/config/environment.go @@ -1,4 +1,4 @@ -package configenv +package config import ( "errors" @@ -21,16 +21,16 @@ import ( type ExportFormat int const ( - //Exports format: Sourceable exports - Exports ExportFormat = iota - //Envfile format: dotenv file - Envfile - //DockerArgs format: --env args for docker - DockerArgs - //Shell format: env arguments for shell - Shell - //Pretty format: pretty-printed in columns - Pretty + //ExportFormatExports format: Sourceable exports + ExportFormatExports ExportFormat = iota + //ExportFormatEnvfile format: dotenv file + ExportFormatEnvfile + //ExportFormatDockerArgs format: --env args for docker + ExportFormatDockerArgs + //ExportFormatShell format: env arguments for shell + ExportFormatShell + //ExportFormatPretty format: pretty-printed in columns + ExportFormatPretty ) //Env is a representation for global or app environment @@ -40,8 +40,8 @@ type Env struct { env map[string]string } -//NewFromString creates an env from the given ENVFILE contents representation -func NewFromString(rep string) (env *Env, err error) { +//newEnvFromString creates an env from the given ENVFILE contents representation +func newEnvFromString(rep string) (env *Env, err error) { envMap, err := godotenv.Unmarshal(rep) env = &Env{ "", @@ -51,8 +51,8 @@ func NewFromString(rep string) (env *Env, err error) { return } -//LoadApp loads an environment for the given app -func LoadApp(appName string) (env *Env, err error) { +//LoadAppEnv loads an environment for the given app +func LoadAppEnv(appName string) (env *Env, err error) { appfile, err := getAppFile(appName) if err != nil { return @@ -60,8 +60,24 @@ func LoadApp(appName string) (env *Env, err error) { return loadFromFile(appName, appfile) } -//LoadGlobal loads the global environmen -func LoadGlobal() (*Env, error) { +//LoadMergedAppEnv loads an app environment merged with the global environment +func LoadMergedAppEnv(appName string) (env *Env, err error) { + env, err = LoadAppEnv(appName) + if err != nil { + return + } + global, err := LoadGlobalEnv() + if err != nil { + common.LogFail(err.Error()) + } + global.Merge(env) + global.filename = "" + global.name = env.name + return global, err +} + +//LoadGlobalEnv loads the global environment +func LoadGlobalEnv() (*Env, error) { return loadFromFile("global", getGlobalFile()) } @@ -142,16 +158,16 @@ func (e *Env) Write() error { //Export the Env in the given format func (e *Env) Export(format ExportFormat) string { switch format { - case Exports: + case ExportFormatExports: return e.ExportfileString() - case Envfile: + case ExportFormatEnvfile: return e.EnvfileString() - case DockerArgs: + case ExportFormatDockerArgs: return e.DockerArgsString() - case Shell: + case ExportFormatShell: return e.ShellString() - case Pretty: - return PrettyPrintEnvEntries("", e.Map()) + case ExportFormatPretty: + return prettyPrintEnvEntries("", e.Map()) default: common.LogFail(fmt.Sprintf("Unknown export format: %v", format)) return "" @@ -166,18 +182,18 @@ func (e *Env) EnvfileString() string { //ExportfileString returns the contents of this Env as bash exports func (e *Env) ExportfileString() string { - return e.stringWithPrefixAndSeparator("export ", "\n", true) + return e.stringWithPrefixAndSeparator("export ", "\n") } //DockerArgsString gets the contents of this Env in the form -env=KEY=VALUE --env... func (e *Env) DockerArgsString() string { - return e.stringWithPrefixAndSeparator("--env=", " ", true) + return e.stringWithPrefixAndSeparator("--env=", " ") } //ShellString gets the contents of this Env in the form "KEY='value' KEY2='value'" // for passing the environment in the shell func (e *Env) ShellString() string { - return e.stringWithPrefixAndSeparator("", " ", true) + return e.stringWithPrefixAndSeparator("", " ") } //ExportBundle writes a tarfile of the environmnet to the given io.Writer. @@ -202,13 +218,25 @@ func (e *Env) ExportBundle(dest io.Writer) error { return nil } -//SingleQuoteEscape escapes the value as if it were shell-quoted in single quotes -func SingleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\''aped' +//stringWithPrefixAndSeparator makes a string of the environment +// with the given prefix and separator for each entry +func (e *Env) stringWithPrefixAndSeparator(prefix string, separator string) string { + keys := e.Keys() + entries := make([]string, len(keys)) + for i, k := range keys { + v := singleQuoteEscape(e.env[k]) + entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, v) + } + return strings.Join(entries, separator) +} + +//singleQuoteEscape escapes the value as if it were shell-quoted in single quotes +func singleQuoteEscape(value string) string { // so that 'esc'apped' -> 'esc'\''aped' return strings.Replace(value, "'", "'\\''", -1) } -//PrettyPrintEnvEntries in columns -func PrettyPrintEnvEntries(prefix string, entries map[string]string) (representation string) { +//prettyPrintEnvEntries in columns +func prettyPrintEnvEntries(prefix string, entries map[string]string) (representation string) { colConfig := columnize.DefaultConfig() colConfig.Prefix = prefix colConfig.Delim = "\x00" @@ -233,21 +261,6 @@ func loadFromFile(name string, filename string) (env *Env, err error) { return } -//stringWithPrefixAndSeparator makes a string of the environment -// with the given prefix and separator for each entry -func (e *Env) stringWithPrefixAndSeparator(prefix string, separator string, allowNewlines bool) string { - keys := e.Keys() - entries := make([]string, len(keys)) - for i, k := range keys { - v := SingleQuoteEscape(e.env[k]) - if !allowNewlines { - v = strings.Replace(v, "\n", "'$'\\n''", -1) - } - entries[i] = fmt.Sprintf("%s%s='%s'", prefix, k, v) - } - return strings.Join(entries, separator) -} - func getAppFile(appName string) (string, error) { err := common.VerifyAppName(appName) if err != nil { diff --git a/plugins/config/environment_test.go b/plugins/config/environment_test.go new file mode 100644 index 000000000..c7bef686f --- /dev/null +++ b/plugins/config/environment_test.go @@ -0,0 +1,102 @@ +package config + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestExportfileRoundtrip(t *testing.T) { + RegisterTestingT(t) + env, err := newEnvFromString("HI='ho'\nFoo='Bar'\n\nBaz='BOFF'") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("Baz", "BOFF", "Foo", "Bar", "HI", "ho"))) + Expect(env.String()).To(Equal("Baz=\"BOFF\"\nFoo=\"Bar\"\nHI=\"ho\"")) + + env, err = newEnvFromString(`export HI="h\no"`) + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("HI", "h\no"))) + Expect(env.String()).To(Equal(`HI="h\no"`)) + + env, err = newEnvFromString("HI='ho'\nFOO=\"'\\nBAR=''\"") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("HI", "ho", "FOO", "'\nBAR=''"))) + Expect(env.String()).To(Equal("FOO=\"'\\nBAR=''\"\nHI=\"ho\"")) + + env, err = newEnvFromString("FOO='bar' ") + Expect(err).NotTo(HaveOccurred()) + Expect(env.Map()).To(Equal(pairs("FOO", "bar"))) + Expect(env.String()).To(Equal(`FOO="bar"`)) + +} + +func TestExportfileErrors(t *testing.T) { + RegisterTestingT(t) + + _, err := newEnvFromString("F\nOO='bar'") //keys cannot have embedded newlines + Expect(err).To(HaveOccurred()) +} + +func TestMerge(t *testing.T) { + RegisterTestingT(t) + e, _ := newEnvFromString("FOO='bar'") + e2, _ := newEnvFromString("BAR='baz'") + e.Merge(e2) + Expect(e.Map()).To(Equal(pairs("BAR", "baz", "FOO", "bar"))) + + e3, _ := newEnvFromString("FOO='ba \\nz'") + e.Merge(e3) + Expect(e.Map()).To(Equal(pairs("BAR", "baz", "FOO", "ba \nz"))) +} + +func TestExport(t *testing.T) { + RegisterTestingT(t) + e, _ := newEnvFromString("BAR='BAZ'\nFOO='b'ar '\nBAZ='a\\nb'") + Expect(e.Export(ExportFormatEnvfile)).To(Equal("BAR=\"BAZ\"\nBAZ=\"a\\nb\"\nFOO=\"b'ar \"")) + Expect(e.Export(ExportFormatDockerArgs)).To(Equal("--env=BAR='BAZ' --env=BAZ='a\nb' --env=FOO='b'\\''ar '")) + Expect(e.Export(ExportFormatShell)).To(Equal("BAR='BAZ' BAZ='a\nb' FOO='b'\\''ar '")) + Expect(e.Export(ExportFormatExports)).To(Equal("export BAR='BAZ'\nexport BAZ='a\nb'\nexport FOO='b'\\''ar '")) +} + +func TestGet(t *testing.T) { + RegisterTestingT(t) + e, err := newEnvFromString("BAR='BAZ'\nFOO='ba\\nr '\nGO='1'\nNOGO='0'") + Expect(err).To(Succeed()) + + v, ok := e.Get("GO") + Expect(ok).To(Equal(true)) + Expect(v).To(Equal("1")) + + v = e.GetDefault("GO", "default") + Expect(v).To(Equal("1")) + + v = e.GetDefault("dne", "default") + Expect(v).To(Equal("default")) + + b := e.GetBoolDefault("dne", true) + Expect(b).To(Equal(true)) + + b = e.GetBoolDefault("dne", false) + Expect(b).To(Equal(false)) + + b = e.GetBoolDefault("GO", false) + Expect(b).To(Equal(true)) + + b = e.GetBoolDefault("NOGO", true) + Expect(b).To(Equal(false)) + + b = e.GetBoolDefault("BAR", false) + Expect(b).To(Equal(true)) //anything but "0" is true +} + +func pairs(vars ...string) map[string]string { + res := map[string]string{} + var i = 0 + for i < len(vars)-1 { + key := vars[i] + value := vars[i+1] + res[key] = value + i += 2 + } + return res +} diff --git a/plugins/config/src/commands/commands.go b/plugins/config/src/commands/commands.go index afbe5246f..1b7f44f40 100644 --- a/plugins/config/src/commands/commands.go +++ b/plugins/config/src/commands/commands.go @@ -7,9 +7,6 @@ import ( "strconv" "strings" - "github.com/dokku/dokku/plugins/common" - "github.com/dokku/dokku/plugins/config/src/configenv" - "github.com/dokku/dokku/plugins/config" columnize "github.com/ryanuber/columnize" ) @@ -43,25 +40,9 @@ func main() { global := args.Bool("global", false, "--global: use the global environment") shell := args.Bool("shell", false, "--shell: in a single-line for usage in command-line utilities [deprecated]") export := args.Bool("export", false, "--export: print the env as eval-compatible exports [deprecated]") + merged := args.Bool("merged", false, "--merged: display the app's envionment merged with the global environment") args.Parse(os.Args[2:]) - appName, _ := config.GetCommonArgs(*global, args.Args()) - env := config.GetConfig(appName, false) - - if *shell && *export { - common.LogFail("Only one of --shell and --export can be given") - } - if *shell { - fmt.Print(env.Export(configenv.Shell)) - } else if *export { - fmt.Println(env.Export(configenv.Exports)) - } else { - contextName := "global" - if appName != "" { - contextName = appName - } - common.LogInfo2(contextName + " config vars") - fmt.Println(env.Export(configenv.Pretty)) - } + config.CommandShow(args.Args(), *global, *shell, *export, *merged) case "config:help": case "help": usage() diff --git a/plugins/config/src/configenv/configenv_test.go b/plugins/config/src/configenv/configenv_test.go deleted file mode 100644 index 9d423e95e..000000000 --- a/plugins/config/src/configenv/configenv_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package configenv - -import ( - "testing" - - . "github.com/onsi/gomega" -) - -func TestExportfileRoundtrip(t *testing.T) { - RegisterTestingT(t) - env, err := NewFromString("HI='ho'\nFoo='Bar'\n\nBaz='BOFF'") - Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("Baz", "BOFF", "Foo", "Bar", "HI", "ho"))) - Expect(env.String()).To(Equal("Baz=\"BOFF\"\nFoo=\"Bar\"\nHI=\"ho\"")) - - env, err = NewFromString(`export HI="h\no"`) - Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "h\no"))) - Expect(env.String()).To(Equal(`HI="h\no"`)) - - env, err = NewFromString("HI='ho'\nFOO=\"'\\nBAR=''\"") - Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("HI", "ho", "FOO", "'\nBAR=''"))) - Expect(env.String()).To(Equal("FOO=\"'\\nBAR=''\"\nHI=\"ho\"")) - - env, err = NewFromString("FOO='bar' ") - Expect(err).NotTo(HaveOccurred()) - Expect(env.Map()).To(Equal(pairs("FOO", "bar"))) - Expect(env.String()).To(Equal(`FOO="bar"`)) - -} - -func TestExportfileErrors(t *testing.T) { - RegisterTestingT(t) - - _, err := NewFromString("F\nOO='bar'") //keys cannot have embedded newlines - Expect(err).To(HaveOccurred()) -} - -func TestMerge(t *testing.T) { - RegisterTestingT(t) - e, _ := NewFromString("FOO='bar'") - e2, _ := NewFromString("BAR='baz'") - e.Merge(e2) - Expect(e.Map()).To(Equal(pairs("BAR", "baz", "FOO", "bar"))) -} - -func TestArrayExport(t *testing.T) { - RegisterTestingT(t) - e, _ := NewFromString("BAR='BAZ'\nFOO='b'ar '") - Expect(e.EnvfileString()).To(Equal("BAR=\"BAZ\"\nFOO=\"b'ar \"")) - Expect(e.DockerArgsString()).To(Equal("--env=BAR='BAZ' --env=FOO='b'\\''ar '")) - Expect(e.ShellString()).To(Equal("BAR='BAZ' FOO='b'\\''ar '")) -} - -func pairs(vars ...string) map[string]string { - res := map[string]string{} - var i = 0 - for i < len(vars)-1 { - key := vars[i] - value := vars[i+1] - res[key] = value - i += 2 - } - return res -} diff --git a/plugins/config/src/subcommands/bundle/bundle.go b/plugins/config/src/subcommands/bundle/bundle.go index fc3b11407..3bb03a16d 100644 --- a/plugins/config/src/subcommands/bundle/bundle.go +++ b/plugins/config/src/subcommands/bundle/bundle.go @@ -2,10 +2,8 @@ package main import ( "flag" - "fmt" "os" - "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" ) @@ -14,11 +12,5 @@ func main() { global := args.Bool("global", false, "--global: use the global environment") merged := args.Bool("merged", false, "--merged: merge app environment and global environment") args.Parse(os.Args[2:]) - - appName, trailingArgs := config.GetCommonArgs(*global, args.Args()) - if len(trailingArgs) > 0 { - common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) - } - config := config.GetConfig(appName, *merged) - config.ExportBundle(os.Stdout) + config.CommandBundle(args.Args(), *global, *merged) } diff --git a/plugins/config/src/subcommands/export/export.go b/plugins/config/src/subcommands/export/export.go index 0c9027fbc..86fd8df8f 100644 --- a/plugins/config/src/subcommands/export/export.go +++ b/plugins/config/src/subcommands/export/export.go @@ -2,12 +2,9 @@ package main import ( "flag" - "fmt" "os" - common "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" - "github.com/dokku/dokku/plugins/config/src/configenv" ) // print the environment to stdout @@ -19,28 +16,5 @@ func main() { merged := args.Bool("merged", false, "--merged: merge app environment and global environment") format := args.String("format", "exports", "--format: [ exports | envfile | docker-args | shell ] which format to export as)") args.Parse(os.Args[2:]) - - appName, trailingArgs := config.GetCommonArgs(*global, args.Args()) - if len(trailingArgs) > 0 { - common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) - } - - env := config.GetConfig(appName, *merged) - exportType := configenv.Exports - suffix := "\n" - switch *format { - case "exports": - exportType = configenv.Exports - case "envfile": - exportType = configenv.Envfile - case "docker-args": - exportType = configenv.DockerArgs - case "shell": - exportType = configenv.Shell - suffix = " " - default: - common.LogFail(fmt.Sprintf("Unknown export format: %v", *format)) - } - exported := env.Export(exportType) - fmt.Print(exported + suffix) + config.CommandExport(args.Args(), *global, *merged, *format) } diff --git a/plugins/config/src/subcommands/get/get.go b/plugins/config/src/subcommands/get/get.go index 6419bb8c7..52840b3fa 100644 --- a/plugins/config/src/subcommands/get/get.go +++ b/plugins/config/src/subcommands/get/get.go @@ -2,12 +2,9 @@ package main import ( "flag" - "fmt" "os" - common "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" - "github.com/dokku/dokku/plugins/config/src/configenv" ) // get the given entries from the specified environment @@ -16,22 +13,5 @@ func main() { global := args.Bool("global", false, "--global: use the global environment") quoted := args.Bool("quoted", false, "--quoted: get the value quoted") args.Parse(os.Args[2:]) - - appName, keys := config.GetCommonArgs(*global, args.Args()) - if len(keys) > 1 { - common.LogFail(fmt.Sprintf("Unexpected argument(s): %v", keys[1:])) - } - if len(keys) == 0 { - common.LogFail("Expected: key") - } - if !config.HasKey(appName, keys[0]) { - os.Exit(1) - } - - value := config.GetWithDefault(appName, keys[0], "") - if *quoted { - fmt.Printf("'%s'", configenv.SingleQuoteEscape(value)) - } else { - fmt.Printf("%s", value) - } + config.CommandGet(args.Args(), *global, *quoted) } diff --git a/plugins/config/src/subcommands/keys/keys.go b/plugins/config/src/subcommands/keys/keys.go index 0694e4dc3..64de1c02b 100644 --- a/plugins/config/src/subcommands/keys/keys.go +++ b/plugins/config/src/subcommands/keys/keys.go @@ -2,10 +2,8 @@ package main import ( "flag" - "fmt" "os" - "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" ) @@ -14,13 +12,5 @@ func main() { global := args.Bool("global", false, "--global: use the global environment") merged := args.Bool("merged", false, "--merged: merge app environment and global environment") args.Parse(os.Args[2:]) - - appName, trailingArgs := config.GetCommonArgs(*global, args.Args()) - if len(trailingArgs) > 0 { - common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) - } - config := config.GetConfig(appName, *merged) - for _, k := range config.Keys() { - fmt.Println(k) - } + config.CommandKeys(args.Args(), *global, *merged) } diff --git a/plugins/config/src/subcommands/set/set.go b/plugins/config/src/subcommands/set/set.go index 276b8dc79..2bee318f7 100644 --- a/plugins/config/src/subcommands/set/set.go +++ b/plugins/config/src/subcommands/set/set.go @@ -1,14 +1,10 @@ package main import ( - "encoding/base64" "flag" - "fmt" "os" - "strings" - common "github.com/dokku/dokku/plugins/common" - config "github.com/dokku/dokku/plugins/config" + "github.com/dokku/dokku/plugins/config" ) // set the given entries from the specified environment @@ -18,23 +14,5 @@ func main() { encoded := args.Bool("encoded", false, "--encoded: interpret VALUEs as base64") noRestart := args.Bool("no-restart", false, "--no-restart: no restart") args.Parse(os.Args[2:]) - appName, pairs := config.GetCommonArgs(*global, args.Args()) - - updated := make(map[string]string) - for _, e := range pairs { - parts := strings.SplitN(e, "=", 2) - if len(parts) == 1 { - common.LogFail("Invalid env pair: " + e) - } - key, value := parts[0], parts[1] - if *encoded { - decoded, err := base64.StdEncoding.DecodeString(value) - if err != nil { - common.LogFail(fmt.Sprintf("%s for key '%s'", err.Error(), key)) - } - value = string(decoded) - } - updated[key] = value - } - config.SetMany(appName, updated, !*noRestart) + config.CommandSet(args.Args(), *global, *noRestart, *encoded) } diff --git a/plugins/config/src/subcommands/unset/unset.go b/plugins/config/src/subcommands/unset/unset.go index 9d388a762..c6c4ee814 100644 --- a/plugins/config/src/subcommands/unset/unset.go +++ b/plugins/config/src/subcommands/unset/unset.go @@ -12,8 +12,6 @@ func main() { args := flag.NewFlagSet("config:unset", flag.ExitOnError) global := args.Bool("global", false, "--global: use the global environment") noRestart := args.Bool("no-restart", false, "--no-restart: no restart") - args.Parse(os.Args[2:]) - appName, keys := config.GetCommonArgs(*global, args.Args()) - config.UnsetMany(appName, keys, !*noRestart) + config.CommandUnset(args.Args(), *global, *noRestart) } diff --git a/plugins/config/subcommands.go b/plugins/config/subcommands.go new file mode 100644 index 000000000..a5b564933 --- /dev/null +++ b/plugins/config/subcommands.go @@ -0,0 +1,167 @@ +package config + +import ( + "encoding/base64" + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +//CommandShow implementes config:show +func CommandShow(args []string, global bool, shell bool, export bool, merged bool) { + appName, _ := getCommonArgs(global, args) + env := getEnvironment(appName, merged) + if shell && export { + common.LogFail("Only one of --shell and --export can be given") + } + if shell { + fmt.Print(env.Export(ExportFormatShell)) + } else if export { + fmt.Println(env.Export(ExportFormatExports)) + } else { + contextName := "global" + if appName != "" { + contextName = appName + } + common.LogInfo2(contextName + " env vars") + fmt.Println(env.Export(ExportFormatPretty)) + } +} + +//CommandGet implements config:get +func CommandGet(args []string, global bool, quoted bool) { + appName, keys := getCommonArgs(global, args) + if len(keys) > 1 { + common.LogFail(fmt.Sprintf("Unexpected argument(s): %v", keys[1:])) + } + if len(keys) == 0 { + common.LogFail("Expected: key") + } + if value, ok := Get(appName, keys[0]); !ok { + os.Exit(1) + } else { + if quoted { + fmt.Printf("'%s'", singleQuoteEscape(value)) + } else { + fmt.Printf("%s", value) + } + } +} + +//CommandUnset implements config:unset +func CommandUnset(args []string, global bool, noRestart bool) { + appName, keys := getCommonArgs(global, args) + err := UnsetMany(appName, keys, !noRestart) + if err != nil { + common.LogFail(err.Error()) + } +} + +//CommandSet implements config:set +func CommandSet(args []string, global bool, noRestart bool, encoded bool) { + appName, pairs := getCommonArgs(global, args) + updated := make(map[string]string) + for _, e := range pairs { + parts := strings.SplitN(e, "=", 2) + if len(parts) == 1 { + common.LogFail("Invalid env pair: " + e) + } + key, value := parts[0], parts[1] + if encoded { + decoded, err := base64.StdEncoding.DecodeString(value) + if err != nil { + common.LogFail(fmt.Sprintf("%s for key '%s'", err.Error(), key)) + } + value = string(decoded) + } + updated[key] = value + } + err := SetMany(appName, updated, !noRestart) + if err != nil { + common.LogFail(err.Error()) + } +} + +//CommandKeys implements config:keys +func CommandKeys(args []string, global bool, merged bool) { + appName, trailingArgs := getCommonArgs(global, args) + if len(trailingArgs) > 0 { + common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) + } + env := getEnvironment(appName, merged) + for _, k := range env.Keys() { + fmt.Println(k) + } +} + +//CommandExport implements config:export +func CommandExport(args []string, global bool, merged bool, format string) { + appName, trailingArgs := getCommonArgs(global, args) + if len(trailingArgs) > 0 { + common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) + } + env := getEnvironment(appName, merged) + exportType := ExportFormatExports + suffix := "\n" + switch format { + case "exports": + exportType = ExportFormatExports + case "envfile": + exportType = ExportFormatEnvfile + case "docker-args": + exportType = ExportFormatDockerArgs + case "shell": + exportType = ExportFormatShell + suffix = " " + case "pretty": + exportType = ExportFormatPretty + default: + common.LogFail(fmt.Sprintf("Unknown export format: %v", format)) + } + exported := env.Export(exportType) + fmt.Print(exported + suffix) +} + +//CommandBundle implements config:bundle +func CommandBundle(args []string, global bool, merged bool) { + appName, trailingArgs := getCommonArgs(global, args) + if len(trailingArgs) > 0 { + common.LogFail(fmt.Sprintf("Trailing argument(s): %v", trailingArgs)) + } + env := getEnvironment(appName, merged) + env.ExportBundle(os.Stdout) +} + +//getEnvironment for the given app (global config if appName is empty). Merge with global environment if merged is true. +func getEnvironment(appName string, merged bool) (env *Env) { + env, err := loadAppOrGlobalEnv(appName) + if err != nil { + common.LogFail(err.Error()) + } + if appName != "" && merged { + env, err = LoadMergedAppEnv(appName) + if err != nil { + common.LogFail(err.Error()) + } + } + return env +} + +//getCommonArgs extracts common positional args (appName and keys) +func getCommonArgs(global bool, args []string) (appName string, keys []string) { + nextArg := 0 + if !global { + if len(args) > 0 { + appName = args[0] + } + if appName == "" { + common.LogFail("Please specify an app or --global") + } else { + nextArg++ + } + } + keys = args[nextArg:] + return appName, keys +} From 1be66bd1bc0ab85abf279c07410258820a13c956 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 16 Oct 2017 02:09:26 -0400 Subject: [PATCH 48/51] [config] don't alias import --- plugins/config/environment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/config/environment.go b/plugins/config/environment.go index 43417cd87..cab3379e8 100644 --- a/plugins/config/environment.go +++ b/plugins/config/environment.go @@ -12,8 +12,8 @@ import ( "os" - common "github.com/dokku/dokku/plugins/common" - godotenv "github.com/joho/godotenv" + "github.com/dokku/dokku/plugins/common" + "github.com/joho/godotenv" "github.com/ryanuber/columnize" ) From 98c45b55b77ff9ec6b8bf49013f3891e1bfb20f0 Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Mon, 16 Oct 2017 02:21:38 -0400 Subject: [PATCH 49/51] fix: newlines at eof --- tests/apps/config/check_deploy | 1 - tests/apps/dockerfile/check_deploy | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/apps/config/check_deploy b/tests/apps/config/check_deploy index 5833d3697..e6b0cb135 100755 --- a/tests/apps/config/check_deploy +++ b/tests/apps/config/check_deploy @@ -14,4 +14,3 @@ base_url="$1" test_url / "config-test" test_url /hello "Hello 'world'" - \ No newline at end of file diff --git a/tests/apps/dockerfile/check_deploy b/tests/apps/dockerfile/check_deploy index bdfcdbea9..c047ec81b 100755 --- a/tests/apps/dockerfile/check_deploy +++ b/tests/apps/dockerfile/check_deploy @@ -13,4 +13,3 @@ base_url="$1" test_url / "Hello World!" test_url /env/HELLO "Hello 'world' " - \ No newline at end of file From 2f7daad200a7bca3fbe064a3abfac1cc7b136b76 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 16 Oct 2017 20:23:30 -0400 Subject: [PATCH 50/51] fix: typo on declare --- plugins/00_dokku-standard/pre-deploy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/00_dokku-standard/pre-deploy b/plugins/00_dokku-standard/pre-deploy index fbe36e544..2c4209955 100755 --- a/plugins/00_dokku-standard/pre-deploy +++ b/plugins/00_dokku-standard/pre-deploy @@ -15,7 +15,7 @@ exec_app_json_scripts() { app_user_pre_deploy_trigger() { local desc="app user pre-deploy trigger" - local APP="$1"; local IMAGE_TAG="$2"; local IMAGE; local DOCKER ARGS + local APP="$1"; local IMAGE_TAG="$2"; local IMAGE; local DOCKER_ARGS local DOKKU_APP_TYPE DOKKU_APP_USER APP_PATHS CONTAINER_PATHS declare -a ARG_ARRAY From 90ee5d58c5901a83c9b24cdbb603e97aa1c2015c Mon Sep 17 00:00:00 2001 From: Alex Quick Date: Tue, 24 Oct 2017 09:11:42 -0400 Subject: [PATCH 51/51] [config] use field names in struct literals --- plugins/config/environment.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/config/environment.go b/plugins/config/environment.go index cab3379e8..1727c6342 100644 --- a/plugins/config/environment.go +++ b/plugins/config/environment.go @@ -44,9 +44,9 @@ type Env struct { func newEnvFromString(rep string) (env *Env, err error) { envMap, err := godotenv.Unmarshal(rep) env = &Env{ - "", - "", - envMap, + name: "", + filename: "", + env: envMap, } return } @@ -254,9 +254,9 @@ func loadFromFile(name string, filename string) (env *Env, err error) { } env = &Env{ - name, - filename, - envMap, + name: name, + filename: filename, + env: envMap, } return }