From 02d6813209dd1e335e7645b68ab70425d7494e44 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 16 Apr 2022 01:29:10 -0400 Subject: [PATCH 01/20] scratch: initial implementation --- plugins/builder-lambda/builder-build | 46 ++++++++++++ plugins/builder-lambda/builder-detect | 17 +++++ plugins/builder-lambda/builder-release | 32 +++++++++ plugins/builder-lambda/commands | 15 ++++ .../dockerfiles/builder-release.Dockerfile | 2 + plugins/builder-lambda/help-functions | 33 +++++++++ plugins/builder-lambda/install | 13 ++++ plugins/builder-lambda/internal-functions | 71 +++++++++++++++++++ plugins/builder-lambda/plugin.toml | 4 ++ plugins/builder-lambda/post-app-clone-setup | 14 ++++ plugins/builder-lambda/post-app-rename-setup | 15 ++++ plugins/builder-lambda/post-delete | 14 ++++ plugins/builder-lambda/report | 6 ++ plugins/builder-lambda/subcommands/default | 6 ++ plugins/builder-lambda/subcommands/report | 6 ++ plugins/builder-lambda/subcommands/set | 30 ++++++++ 16 files changed, 324 insertions(+) create mode 100755 plugins/builder-lambda/builder-build create mode 100755 plugins/builder-lambda/builder-detect create mode 100755 plugins/builder-lambda/builder-release create mode 100755 plugins/builder-lambda/commands create mode 100644 plugins/builder-lambda/dockerfiles/builder-release.Dockerfile create mode 100755 plugins/builder-lambda/help-functions create mode 100755 plugins/builder-lambda/install create mode 100755 plugins/builder-lambda/internal-functions create mode 100644 plugins/builder-lambda/plugin.toml create mode 100755 plugins/builder-lambda/post-app-clone-setup create mode 100755 plugins/builder-lambda/post-app-rename-setup create mode 100755 plugins/builder-lambda/post-delete create mode 100755 plugins/builder-lambda/report create mode 100755 plugins/builder-lambda/subcommands/default create mode 100755 plugins/builder-lambda/subcommands/report create mode 100755 plugins/builder-lambda/subcommands/set diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build new file mode 100755 index 000000000..547556fb2 --- /dev/null +++ b/plugins/builder-lambda/builder-build @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_AVAILABLE_PATH/config/functions" +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-builder-build() { + declare desc="builder-lambda builder-build plugin trigger" + declare trigger="builder-build" + declare BUILDER_TYPE="$1" APP="$2" SOURCECODE_WORK_DIR="$3" + + if [[ "$BUILDER_TYPE" != "lambda" ]]; then + return + fi + + # write lambda file out + + dokku_log_info1 "Building $APP from Dockerfile" + + local IMAGE=$(get_app_image_name "$APP") + local DOCKER_BUILD_LABEL_ARGS="--label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.app-name=$APP" + + pushd "$SOURCECODE_WORK_DIR" &>/dev/null + + # extract first port from Dockerfile + suppress_output dos2unix Dockerfile + local DOCKERFILE_PORTS=$(get_dockerfile_exposed_ports Dockerfile) + [[ -n "$DOCKERFILE_PORTS" ]] && config_set --no-restart "$APP" DOKKU_DOCKERFILE_PORTS="$DOCKERFILE_PORTS" + plugn trigger pre-build-lambda "$APP" + + [[ "$DOKKU_DOCKERFILE_CACHE_BUILD" == "false" ]] && DOKKU_DOCKER_BUILD_OPTS="$DOKKU_DOCKER_BUILD_OPTS --no-cache" + local DOCKER_ARGS=$(: | plugn trigger docker-args-build "$APP" "$BUILDER_TYPE") + DOCKER_ARGS+=$(: | plugn trigger docker-args-process-build "$APP" "$BUILDER_TYPE") + + # strip --link, --volume and -v args from DOCKER_ARGS + local DOCKER_ARGS=$(sed -e "s/^--link=[[:graph:]]\+[[:blank:]]\?//g" -e "s/^--link[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" -e "s/^--volume=[[:graph:]]\+[[:blank:]]\?//g" -e "s/^--volume[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" -e "s/^-v[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" <<<"$DOCKER_ARGS") + declare -a ARG_ARRAY + eval "ARG_ARRAY=($DOCKER_ARGS)" + + "$DOCKER_BIN" image build $DOCKER_BUILD_LABEL_ARGS $DOKKU_GLOBAL_BUILD_ARGS "${ARG_ARRAY[@]}" "${DOKKU_DOCKER_BUILD_OPTS[@]}" -t $IMAGE . + + plugn trigger post-build-lambda "$APP" +} + +trigger-builder-lambda-builder-build "$@" diff --git a/plugins/builder-lambda/builder-detect b/plugins/builder-lambda/builder-detect new file mode 100755 index 000000000..1f87e2ae3 --- /dev/null +++ b/plugins/builder-lambda/builder-detect @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/config/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-builder-detect() { + declare desc="builder-lambda builder-detect plugin trigger" + declare trigger="builder-detect" + declare APP="$1" SOURCECODE_WORK_DIR="$2" + + if [[ -f "$SOURCECODE_WORK_DIR/lambda.yml" ]]; then + echo "lambda" + return + fi +} + +trigger-builder-lambda-builder-detect "$@" diff --git a/plugins/builder-lambda/builder-release b/plugins/builder-lambda/builder-release new file mode 100755 index 000000000..d77bd95a2 --- /dev/null +++ b/plugins/builder-lambda/builder-release @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" + +trigger-builder-lambda-builder-release() { + declare desc="builder-lambda builder-release plugin trigger" + declare trigger="builder-release" + declare BUILDER_TYPE="$1" APP="$2" IMAGE_TAG="$3" + + if [[ "$BUILDER_TYPE" != "lambda" ]]; then + return + fi + + local DOCKER_BUILD_LABEL_ARGS="--label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.app-name=$APP --label=com.dokku.image-stage=release --label=dokku" + + plugn trigger pre-release-lambda "$APP" "$IMAGE_TAG" + + TMP_WORK_DIR="$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")" + trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN INT TERM EXIT + + local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") + if ! suppress_output "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-lambda/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t $IMAGE "$TMP_WORK_DIR"; then + dokku_log_warn "Failure injecting docker labels on image" + return 1 + fi + + plugn trigger post-release-lambda "$APP" "$IMAGE_TAG" + plugn trigger post-release-builder "$BUILDER_TYPE" "$APP" "$IMAGE" +} + +trigger-builder-lambda-builder-release "$@" diff --git a/plugins/builder-lambda/commands b/plugins/builder-lambda/commands new file mode 100755 index 000000000..0e355afce --- /dev/null +++ b/plugins/builder-lambda/commands @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/help-functions" + +case "$1" in + help | builder-lambda:help) + cmd-builder-lambda-help "$@" + ;; + + *) + exit "$DOKKU_NOT_IMPLEMENTED_EXIT" + ;; + +esac diff --git a/plugins/builder-lambda/dockerfiles/builder-release.Dockerfile b/plugins/builder-lambda/dockerfiles/builder-release.Dockerfile new file mode 100644 index 000000000..cca256f21 --- /dev/null +++ b/plugins/builder-lambda/dockerfiles/builder-release.Dockerfile @@ -0,0 +1,2 @@ +ARG APP_IMAGE +FROM $APP_IMAGE diff --git a/plugins/builder-lambda/help-functions b/plugins/builder-lambda/help-functions new file mode 100755 index 000000000..341fe44ac --- /dev/null +++ b/plugins/builder-lambda/help-functions @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-help() { + declare desc="help command" + declare CMD="$1" + local plugin_name="builder-lambda" + local plugin_description="Manage the lambda builder integration for an app" + + if [[ "$CMD" == "${plugin_name}:help" ]]; then + echo -e "Usage: dokku ${plugin_name}[:COMMAND]" + echo '' + echo "$plugin_description" + echo '' + echo 'Additional commands:' + fn-help-content | sort | column -c2 -t -s, + elif [[ $(ps -o command= $PPID) == *"--all"* ]]; then + fn-help-content + else + cat <] [], Displays a builder-lambda report for one or more apps + builder-lambda:set (), Set or clear a builder-lambda property for an app +help_content +} diff --git a/plugins/builder-lambda/install b/plugins/builder-lambda/install new file mode 100755 index 000000000..b3fd166e8 --- /dev/null +++ b/plugins/builder-lambda/install @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-install() { + declare desc="installs the builder-lambda plugin" + declare trigger="install" + + fn-plugin-property-setup "builder-lambda" +} + +trigger-builder-lambda-install "$@" diff --git a/plugins/builder-lambda/internal-functions b/plugins/builder-lambda/internal-functions new file mode 100755 index 000000000..136fab6cf --- /dev/null +++ b/plugins/builder-lambda/internal-functions @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-report() { + declare desc="displays a builder-lambda report for one or more apps" + declare cmd="builder-lambda:report" + [[ "$1" == "$cmd" ]] && shift 1 + declare APP="$1" INFO_FLAG="$2" + + if [[ -n "$APP" ]] && [[ "$APP" == --* ]]; then + INFO_FLAG="$APP" + APP="" + fi + + if [[ -z "$APP" ]] && [[ -z "$INFO_FLAG" ]]; then + INFO_FLAG="true" + fi + + if [[ -z "$APP" ]]; then + for app in $(dokku_apps); do + cmd-builder-lambda-report-single "$app" "$INFO_FLAG" | tee || true + done + else + cmd-builder-lambda-report-single "$APP" "$INFO_FLAG" + fi +} + +cmd-builder-lambda-report-single() { + declare APP="$1" INFO_FLAG="$2" + if [[ "$INFO_FLAG" == "true" ]]; then + INFO_FLAG="" + fi + verify_app_name "$APP" + local flag_map=( + "--builder-lambda-language: $(fn-builder-lambda-language "$APP")" + ) + + if [[ -z "$INFO_FLAG" ]]; then + dokku_log_info2_quiet "${APP} builder-lambda information" + for flag in "${flag_map[@]}"; do + key="$(echo "${flag#--}" | cut -f1 -d' ' | tr - ' ')" + dokku_log_verbose "$(printf "%-30s %-25s" "${key^}" "${flag#*: }")" + done + else + local match=false + local value_exists=false + for flag in "${flag_map[@]}"; do + valid_flags="${valid_flags} $(echo "$flag" | cut -d':' -f1)" + if [[ "$flag" == "${INFO_FLAG}:"* ]]; then + value=${flag#*: } + size="${#value}" + if [[ "$size" -ne 0 ]]; then + echo "$value" && match=true && value_exists=true + else + match=true + fi + fi + done + [[ "$match" == "true" ]] || dokku_log_fail "Invalid flag passed, valid flags:${valid_flags}" + [[ "$value_exists" == "true" ]] || dokku_log_fail "not deployed" + fi +} + +fn-builder-lambda-language() { + declare APP="$1" + + fn-plugin-property-get "builder-lambda" "$APP" "language" "" +} diff --git a/plugins/builder-lambda/plugin.toml b/plugins/builder-lambda/plugin.toml new file mode 100644 index 000000000..223a9dd01 --- /dev/null +++ b/plugins/builder-lambda/plugin.toml @@ -0,0 +1,4 @@ +[plugin] +description = "dokku core builder-lambda plugin" +version = "0.27.0" +[plugin.config] diff --git a/plugins/builder-lambda/post-app-clone-setup b/plugins/builder-lambda/post-app-clone-setup new file mode 100755 index 000000000..086a05200 --- /dev/null +++ b/plugins/builder-lambda/post-app-clone-setup @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-post-app-clone-setup() { + declare desc="sets up properties for new app" + declare trigger="post-app-clone-setup" + declare OLD_APP="$1" NEW_APP="$2" + + fn-plugin-property-clone "builder-lambda" "$OLD_APP" "$NEW_APP" +} + +trigger-builder-lambda-post-app-clone-setup "$@" diff --git a/plugins/builder-lambda/post-app-rename-setup b/plugins/builder-lambda/post-app-rename-setup new file mode 100755 index 000000000..c947a84b3 --- /dev/null +++ b/plugins/builder-lambda/post-app-rename-setup @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-post-app-rename-setup() { + declare desc="updates settings for new app" + declare trigger="post-app-rename-setup" + declare OLD_APP="$1" NEW_APP="$2" + + fn-plugin-property-clone "builder-lambda" "$OLD_APP" "$NEW_APP" + fn-plugin-property-destroy "builder-lambda" "$OLD_APP" +} + +trigger-builder-lambda-post-app-rename-setup "$@" diff --git a/plugins/builder-lambda/post-delete b/plugins/builder-lambda/post-delete new file mode 100755 index 000000000..ce2d88aa7 --- /dev/null +++ b/plugins/builder-lambda/post-delete @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-post-delete() { + declare desc="destroys the builder-lambda properties for a given app" + declare trigger="post-delete" + declare APP="$1" + + fn-plugin-property-destroy "builder-lambda" "$APP" +} + +trigger-builder-lambda-post-delete "$@" diff --git a/plugins/builder-lambda/report b/plugins/builder-lambda/report new file mode 100755 index 000000000..3271b3684 --- /dev/null +++ b/plugins/builder-lambda/report @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-report-single "$@" diff --git a/plugins/builder-lambda/subcommands/default b/plugins/builder-lambda/subcommands/default new file mode 100755 index 000000000..124c65ef9 --- /dev/null +++ b/plugins/builder-lambda/subcommands/default @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/help-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-help "builder-lambda:help" diff --git a/plugins/builder-lambda/subcommands/report b/plugins/builder-lambda/subcommands/report new file mode 100755 index 000000000..9f56cd32f --- /dev/null +++ b/plugins/builder-lambda/subcommands/report @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-report "$@" diff --git a/plugins/builder-lambda/subcommands/set b/plugins/builder-lambda/subcommands/set new file mode 100755 index 000000000..554742e8d --- /dev/null +++ b/plugins/builder-lambda/subcommands/set @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-set() { + declare desc="set or clear a builder-lambda property for an app" + declare cmd="builder-lambda:set" + [[ "$1" == "$cmd" ]] && shift 1 + declare APP="$1" KEY="$2" VALUE="$3" + local VALID_KEYS=("language") + [[ "$APP" == "--global" ]] || verify_app_name "$APP" + + [[ -z "$KEY" ]] && dokku_log_fail "No key specified" + + if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then + dokku_log_fail "Invalid key specified, valid keys include: language" + fi + + if [[ -n "$VALUE" ]]; then + dokku_log_info2_quiet "Setting ${KEY} to ${VALUE}" + fn-plugin-property-write "builder-lambda" "$APP" "$KEY" "$VALUE" + else + dokku_log_info2_quiet "Unsetting ${KEY}" + fn-plugin-property-delete "builder-lambda" "$APP" "$KEY" + fi +} + +cmd-builder-lambda-set "$@" From 92790b70e8cc6e9f2ff48eec1283fe1860364bf8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 17 Apr 2022 02:08:48 -0400 Subject: [PATCH 02/20] feat: broken support for lambda-builder It's currently broken because there are flags in there that aren't actually implemented --- debian/control | 2 +- plugins/builder-lambda/builder-build | 24 ++++-------------- plugins/builder-lambda/builder-release | 4 +-- plugins/builder-lambda/core-post-extract | 30 +++++++++++++++++++++++ plugins/builder-lambda/internal-functions | 26 +++++++++++++++++--- plugins/builder-lambda/subcommands/set | 4 +-- 6 files changed, 63 insertions(+), 27 deletions(-) create mode 100755 plugins/builder-lambda/core-post-extract diff --git a/debian/control b/debian/control index b872c85c5..6f21771cf 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.27.10 Section: web Priority: optional Architecture: amd64 -Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, netrc, software-properties-common, parallel, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip +Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip Recommends: herokuish (>= 0.3.4), bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build index 547556fb2..c564ba227 100755 --- a/plugins/builder-lambda/builder-build +++ b/plugins/builder-lambda/builder-build @@ -1,7 +1,5 @@ #!/usr/bin/env bash source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/config/functions" -source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" set -eo pipefail [[ $DOKKU_TRACE ]] && set -x @@ -14,31 +12,19 @@ trigger-builder-lambda-builder-build() { return fi - # write lambda file out - - dokku_log_info1 "Building $APP from Dockerfile" + dokku_log_info1 "Building $APP from lambda" local IMAGE=$(get_app_image_name "$APP") - local DOCKER_BUILD_LABEL_ARGS="--label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.app-name=$APP" pushd "$SOURCECODE_WORK_DIR" &>/dev/null - # extract first port from Dockerfile - suppress_output dos2unix Dockerfile - local DOCKERFILE_PORTS=$(get_dockerfile_exposed_ports Dockerfile) - [[ -n "$DOCKERFILE_PORTS" ]] && config_set --no-restart "$APP" DOKKU_DOCKERFILE_PORTS="$DOCKERFILE_PORTS" plugn trigger pre-build-lambda "$APP" - [[ "$DOKKU_DOCKERFILE_CACHE_BUILD" == "false" ]] && DOKKU_DOCKER_BUILD_OPTS="$DOKKU_DOCKER_BUILD_OPTS --no-cache" - local DOCKER_ARGS=$(: | plugn trigger docker-args-build "$APP" "$BUILDER_TYPE") - DOCKER_ARGS+=$(: | plugn trigger docker-args-process-build "$APP" "$BUILDER_TYPE") + # shellcheck disable=SC2086 + lambda-builder build --build-image --generate-procfile --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" - # strip --link, --volume and -v args from DOCKER_ARGS - local DOCKER_ARGS=$(sed -e "s/^--link=[[:graph:]]\+[[:blank:]]\?//g" -e "s/^--link[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" -e "s/^--volume=[[:graph:]]\+[[:blank:]]\?//g" -e "s/^--volume[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" -e "s/^-v[[:blank:]]\?[[:graph:]]\+[[:blank:]]\?//g" <<<"$DOCKER_ARGS") - declare -a ARG_ARRAY - eval "ARG_ARRAY=($DOCKER_ARGS)" - - "$DOCKER_BIN" image build $DOCKER_BUILD_LABEL_ARGS $DOKKU_GLOBAL_BUILD_ARGS "${ARG_ARRAY[@]}" "${DOKKU_DOCKER_BUILD_OPTS[@]}" -t $IMAGE . + # todo: extract lambda.zip and put it somewhere for safe-keeping + # todo: write procfile to shared procfile location? plugn trigger post-build-lambda "$APP" } diff --git a/plugins/builder-lambda/builder-release b/plugins/builder-lambda/builder-release index d77bd95a2..087cb8007 100755 --- a/plugins/builder-lambda/builder-release +++ b/plugins/builder-lambda/builder-release @@ -1,7 +1,7 @@ #!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" set -eo pipefail [[ $DOKKU_TRACE ]] && set -x -source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" trigger-builder-lambda-builder-release() { declare desc="builder-lambda builder-release plugin trigger" @@ -20,7 +20,7 @@ trigger-builder-lambda-builder-release() { trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN INT TERM EXIT local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") - if ! suppress_output "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-lambda/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t $IMAGE "$TMP_WORK_DIR"; then + if ! suppress_output "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-lambda/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t "$IMAGE" "$TMP_WORK_DIR"; then dokku_log_warn "Failure injecting docker labels on image" return 1 fi diff --git a/plugins/builder-lambda/core-post-extract b/plugins/builder-lambda/core-post-extract new file mode 100755 index 000000000..bc48c6230 --- /dev/null +++ b/plugins/builder-lambda/core-post-extract @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_AVAILABLE_PATH/builder-pack/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-core-post-extract() { + declare desc="builder-lambda post-extract plugin trigger" + declare trigger="post-extract" + declare APP="$1" SOURCECODE_WORK_DIR="$2" + local NEW_PROJECT_TOML="$(fn-builder-lambda-computed-lambdayml-path "$APP")" + + pushd "$TMP_WORK_DIR" >/dev/null + + if [[ -z "$NEW_LAMBDA_YML" ]]; then + return + fi + + if [[ ! -f "$NEW_LAMBDA_YML" ]]; then + rm -f lambda.yml + return + fi + + if [[ "$NEW_PROJECT_TOML" != "lambda.yml" ]]; then + mv "$NEW_PROJECT_TOML" lambda.yml + fi + popd &>/dev/null || pushd "/tmp" >/dev/null +} + +trigger-builder-lambda-core-post-extract "$@" diff --git a/plugins/builder-lambda/internal-functions b/plugins/builder-lambda/internal-functions index 136fab6cf..f5dc8a7b6 100755 --- a/plugins/builder-lambda/internal-functions +++ b/plugins/builder-lambda/internal-functions @@ -35,7 +35,9 @@ cmd-builder-lambda-report-single() { fi verify_app_name "$APP" local flag_map=( - "--builder-lambda-language: $(fn-builder-lambda-language "$APP")" + "--builder-lambda-computed-lambdayml-path: $(fn-builder-lambda-computed-lambdayml-path "$APP")" + "--builder-lambda-global-lambdayml-path: $(fn-builder-lambda-global-lambdayml-path "$APP")" + "--builder-lambda-lambdayml-path: $(fn-builder-lambda-lambdayml-path "$APP")" ) if [[ -z "$INFO_FLAG" ]]; then @@ -64,8 +66,26 @@ cmd-builder-lambda-report-single() { fi } -fn-builder-lambda-language() { + +fn-builder-lambda-computed-lambdayml-path() { declare APP="$1" - fn-plugin-property-get "builder-lambda" "$APP" "language" "" + file="$(fn-builder-lambda-lambdayml-path "$APP")" + if [[ "$file" == "" ]]; then + file="$(fn-builder-lambda-global-lambdayml-path "$APP")" + fi + + echo "$file" +} + +fn-builder-lambda-global-lambdayml-path() { + declare APP="$1" + + fn-plugin-property-get "builder-lambda" "--global" "lambdayml-path" "lambda.yml" +} + +fn-builder-lambda-lambdayml-path() { + declare APP="$1" + + fn-plugin-property-get "builder-lambda" "$APP" "lambdayml-path" "" } diff --git a/plugins/builder-lambda/subcommands/set b/plugins/builder-lambda/subcommands/set index 554742e8d..fcbe16954 100755 --- a/plugins/builder-lambda/subcommands/set +++ b/plugins/builder-lambda/subcommands/set @@ -9,13 +9,13 @@ cmd-builder-lambda-set() { declare cmd="builder-lambda:set" [[ "$1" == "$cmd" ]] && shift 1 declare APP="$1" KEY="$2" VALUE="$3" - local VALID_KEYS=("language") + local VALID_KEYS=("lambdayml-path") [[ "$APP" == "--global" ]] || verify_app_name "$APP" [[ -z "$KEY" ]] && dokku_log_fail "No key specified" if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then - dokku_log_fail "Invalid key specified, valid keys include: language" + dokku_log_fail "Invalid key specified, valid keys include: lambdayml-path" fi if [[ -n "$VALUE" ]]; then From 3a09d2217f3428abda776743a414082dc37ef573 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 17 Apr 2022 02:10:10 -0400 Subject: [PATCH 03/20] feat: initial templates for scheduling (not building, but i need them somewhere) --- .../templates/samconfig.toml.sigil | 13 ++++++++++ .../templates/template.yaml.sigil | 25 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 plugins/builder-lambda/templates/samconfig.toml.sigil create mode 100644 plugins/builder-lambda/templates/template.yaml.sigil diff --git a/plugins/builder-lambda/templates/samconfig.toml.sigil b/plugins/builder-lambda/templates/samconfig.toml.sigil new file mode 100644 index 000000000..9af08445b --- /dev/null +++ b/plugins/builder-lambda/templates/samconfig.toml.sigil @@ -0,0 +1,13 @@ +version = 0.1 + +[default] +[default.deploy] +[default.deploy.parameters] +stack_name = "{{ $.APP }}-lambda" +s3_bucket = "{{ $.S3_BUCKET }}" +s3_prefix = "{{ $.APP }}" +region = "{{ $.AWS_REGION }}" +confirm_changeset = true +fail_on_empty_changeset = false +capabilities = "CAPABILITY_IAM" +image_repositories = [] diff --git a/plugins/builder-lambda/templates/template.yaml.sigil b/plugins/builder-lambda/templates/template.yaml.sigil new file mode 100644 index 000000000..dab06e7bc --- /dev/null +++ b/plugins/builder-lambda/templates/template.yaml.sigil @@ -0,0 +1,25 @@ +--- +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: > + {{ $.APP }} + +# todo: iterate over every process type and create a resource entry +Resources: + {{ $.APP}}{{ $.PROCESS_TYPE | capitalize}}: + Type: AWS::Serverless::Function + Properties: + Architectures: + - x86_64 + CodeUri: s3://{{ $.S3_BUCKET }}/{{ $.APP }}/{{ $.GIT_REV }} + FunctionName: {{ $.APP | lower }}{{ $.PROCESS_TYPE | lower }} + Handler: {{ $.HANDLER }} + Runtime: {{ $.RUNTIME }} + Role: {{ $.IAM_ROLE }} + Timeout: 3 + Environment: + Variables: + {{ range $key, $value := . }} + {{ $key }}: | + {{ $value }} + {{ end }} From f5499347403f6967f16b0306a9473e78800f67a2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 23 Apr 2022 02:31:00 -0400 Subject: [PATCH 04/20] fix: update lambda-builder usage and write out Procfile/lambda.zip --- plugins/builder-lambda/builder-build | 17 ++++++++++++++--- plugins/builder-lambda/install | 3 +++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build index c564ba227..4f709c7d5 100755 --- a/plugins/builder-lambda/builder-build +++ b/plugins/builder-lambda/builder-build @@ -21,10 +21,21 @@ trigger-builder-lambda-builder-build() { plugn trigger pre-build-lambda "$APP" # shellcheck disable=SC2086 - lambda-builder build --build-image --generate-procfile --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" + lambda-builder build --generate-image --write-procfile --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" + if [[ ! -f "$SOURCECODE_WORK_DIR/lambda.zip" ]]; then + dokku_log_warn "Compressed lambda.zip not detected, failed to build lambda function" + return 1 + fi - # todo: extract lambda.zip and put it somewhere for safe-keeping - # todo: write procfile to shared procfile location? + local GIT_REV="$(plugn trigger git-revision "$APP")" + pusd "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" >/dev/null + rm -f -- *.zip + popd &>/dev/null || pushd "/tmp" >/dev/null + cp "$SOURCECODE_WORK_DIR/lambda.zip" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.zip" + if [[ -f "$SOURCECODE_WORK_DIR/Procfile" ]]; then + cp "$SOURCECODE_WORK_DIR/lambda.zip" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.zip" + cp "$SOURCECODE_WORK_DIR/Procfile" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.Procfile" + fi plugn trigger post-build-lambda "$APP" } diff --git a/plugins/builder-lambda/install b/plugins/builder-lambda/install index b3fd166e8..66c8aa98d 100755 --- a/plugins/builder-lambda/install +++ b/plugins/builder-lambda/install @@ -8,6 +8,9 @@ trigger-builder-lambda-install() { declare trigger="install" fn-plugin-property-setup "builder-lambda" + + mkdir -p "${DOKKU_LIB_ROOT}/data/builder-lambda" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/builder-lambda" } trigger-builder-lambda-install "$@" From f2e791f48fa67c9108ef5ca563981e70e9c48b33 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 29 May 2022 16:51:56 -0400 Subject: [PATCH 05/20] fix: do not detect builder-lambda on armhf While the lambda-builder project builds for armhf, the underlying images are not armhf/arm compatible (only arm64). --- plugins/builder-lambda/builder-detect | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/builder-lambda/builder-detect b/plugins/builder-lambda/builder-detect index 1f87e2ae3..930c41ef1 100755 --- a/plugins/builder-lambda/builder-detect +++ b/plugins/builder-lambda/builder-detect @@ -7,6 +7,11 @@ trigger-builder-lambda-builder-detect() { declare desc="builder-lambda builder-detect plugin trigger" declare trigger="builder-detect" declare APP="$1" SOURCECODE_WORK_DIR="$2" + local ARCHITECTURE="$(dpkg --print-architecture 2>/dev/null || true)" + + if [[ "$ARCHITECTURE" == "armhf" ]]; then + return + fi if [[ -f "$SOURCECODE_WORK_DIR/lambda.yml" ]]; then echo "lambda" From 8ca71d9ff50defa15d54959584f259be54a473ba Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 29 May 2022 17:23:33 -0400 Subject: [PATCH 06/20] fix: add com.dokku.builder-type label to built image This will allow schedulers to make decisions when interacting with images from specific builders. --- plugins/builder-lambda/builder-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build index 4f709c7d5..f17af7ffe 100755 --- a/plugins/builder-lambda/builder-build +++ b/plugins/builder-lambda/builder-build @@ -21,7 +21,7 @@ trigger-builder-lambda-builder-build() { plugn trigger pre-build-lambda "$APP" # shellcheck disable=SC2086 - lambda-builder build --generate-image --write-procfile --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" + lambda-builder build --generate-image --write-procfile --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.image-stage=build --label=com.dokku.builder-type=lambda "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" if [[ ! -f "$SOURCECODE_WORK_DIR/lambda.zip" ]]; then dokku_log_warn "Compressed lambda.zip not detected, failed to build lambda function" return 1 From 951dd98f17818afb21485dc99cb22bc7ea73cf33 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 29 May 2022 17:41:54 -0400 Subject: [PATCH 07/20] feat: ensure the container is routable by dokku - Keep the container alive via DOCKER_LAMBDA_STAY_OPEN=1 - Set the default listening port to 5000 (which is what herokuish will use) --- plugins/builder-lambda/builder-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build index f17af7ffe..fdf226a85 100755 --- a/plugins/builder-lambda/builder-build +++ b/plugins/builder-lambda/builder-build @@ -21,7 +21,7 @@ trigger-builder-lambda-builder-build() { plugn trigger pre-build-lambda "$APP" # shellcheck disable=SC2086 - lambda-builder build --generate-image --write-procfile --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.image-stage=build --label=com.dokku.builder-type=lambda "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" + lambda-builder build --generate-image --write-procfile --image-env=DOCKER_LAMBDA_STAY_OPEN=1 --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.image-stage=build --label=com.dokku.builder-type=lambda "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --port 5000 --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" if [[ ! -f "$SOURCECODE_WORK_DIR/lambda.zip" ]]; then dokku_log_warn "Compressed lambda.zip not detected, failed to build lambda function" return 1 From 2715d2a4851935645414052a3a1971e7d7c6c6dd Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 29 May 2022 18:01:35 -0400 Subject: [PATCH 08/20] chore: drop extra cp statement --- plugins/builder-lambda/builder-build | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build index fdf226a85..a396e577f 100755 --- a/plugins/builder-lambda/builder-build +++ b/plugins/builder-lambda/builder-build @@ -33,7 +33,6 @@ trigger-builder-lambda-builder-build() { popd &>/dev/null || pushd "/tmp" >/dev/null cp "$SOURCECODE_WORK_DIR/lambda.zip" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.zip" if [[ -f "$SOURCECODE_WORK_DIR/Procfile" ]]; then - cp "$SOURCECODE_WORK_DIR/lambda.zip" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.zip" cp "$SOURCECODE_WORK_DIR/Procfile" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.Procfile" fi From 7eecbefdd0cfa6723f40c5d9a317058264eeff2d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 29 May 2022 18:02:53 -0400 Subject: [PATCH 09/20] chore: run shfmt --- plugins/builder-lambda/internal-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/builder-lambda/internal-functions b/plugins/builder-lambda/internal-functions index f5dc8a7b6..d9341ae88 100755 --- a/plugins/builder-lambda/internal-functions +++ b/plugins/builder-lambda/internal-functions @@ -66,7 +66,6 @@ cmd-builder-lambda-report-single() { fi } - fn-builder-lambda-computed-lambdayml-path() { declare APP="$1" From 35c248fb5c11908b0bd8c65a5ccb080e1505d3c3 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 29 May 2022 18:08:37 -0400 Subject: [PATCH 10/20] fix: use correct variable name for lambda.yml path --- plugins/builder-lambda/core-post-extract | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/builder-lambda/core-post-extract b/plugins/builder-lambda/core-post-extract index bc48c6230..3dc631426 100755 --- a/plugins/builder-lambda/core-post-extract +++ b/plugins/builder-lambda/core-post-extract @@ -8,7 +8,7 @@ trigger-builder-lambda-core-post-extract() { declare desc="builder-lambda post-extract plugin trigger" declare trigger="post-extract" declare APP="$1" SOURCECODE_WORK_DIR="$2" - local NEW_PROJECT_TOML="$(fn-builder-lambda-computed-lambdayml-path "$APP")" + local NEW_LAMBDA_YML="$(fn-builder-lambda-computed-lambdayml-path "$APP")" pushd "$TMP_WORK_DIR" >/dev/null @@ -21,8 +21,8 @@ trigger-builder-lambda-core-post-extract() { return fi - if [[ "$NEW_PROJECT_TOML" != "lambda.yml" ]]; then - mv "$NEW_PROJECT_TOML" lambda.yml + if [[ "$NEW_LAMBDA_YML" != "lambda.yml" ]]; then + mv "$NEW_LAMBDA_YML" lambda.yml fi popd &>/dev/null || pushd "/tmp" >/dev/null } From fbd77ee5e88bfd4b914988cffada1e2897051511 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 29 May 2022 18:18:36 -0400 Subject: [PATCH 11/20] docs: document the lambda builder --- .../deployment/builders/builder-management.md | 1 + docs/deployment/builders/lambda.md | 107 ++++++++++++++++++ docs/template.html | 1 + 3 files changed, 109 insertions(+) create mode 100644 docs/deployment/builders/lambda.md diff --git a/docs/deployment/builders/builder-management.md b/docs/deployment/builders/builder-management.md index e15e239b8..4c383751b 100644 --- a/docs/deployment/builders/builder-management.md +++ b/docs/deployment/builders/builder-management.md @@ -17,6 +17,7 @@ Dokku supports the following built-in builders: - `builder-dockerfile`: Builds apps using a `Dockerfile` via `docker build`. See the [dockerfile builder documentation](/docs/deployment/builders/dockerfiles.md) for more information on how this builder functions. - `builder-herokuish`: Builds apps with Heroku's v2a Buildpack specification via `gliderlabs/herokuish`. See the [herokuish builder documentation](/docs/deployment/builders/herokuish-buildpacks.md) for more information on how this builder functions. +- `builder-lambda`: Builds AWS Lambda functions in an environment simulating AWS Lambda runtimes via [lambda-builder](https://github.com/dokku/lambda-builder). See the [lambda builder documentation](/docs/deployment/builders/lambda.md) for more information on how this builder functions. - `builder-null`: Does nothing during the build phase. See the [null builder documentation](/docs/deployment/builders/null.md) for more information on how this builder functions. - `builder-pack`: Builds apps with Cloud Native Buildpacks via the `pack-cli` tool. See the [cloud native buildpacks builder documentation](/docs/deployment/builders/cloud-native-buildpacks.md) for more information on how this builder functions. diff --git a/docs/deployment/builders/lambda.md b/docs/deployment/builders/lambda.md new file mode 100644 index 000000000..b2161d2b4 --- /dev/null +++ b/docs/deployment/builders/lambda.md @@ -0,0 +1,107 @@ +# Lambda Builder + +> New as of 0.28.0 + +The `lambda` builder builds AWS Lambda functions in an environment simulating AWS Lambda runtimes via [lambda-builder](https://github.com/dokku/lambda-builder). Apps built via this builder can run natively in Dokku and may also have their artifacts scheduled to Lambda via an appropriate scheduler. + +## Usage + +### Detection + +This builder will be auto-detected in the following case: + +- A `lambda.yml` exists in the root of the app repository. + +The builder may also be selected via the `builder:set` command + +```shell +dokku builder:set node-js-app selected lambda +``` + +### Supported languages + +The `lambda` builder plugin supports the following AWS runtime languages on Amazon Linux 2: + +- dotnet +- go (builder is based on AL1) +- nodejs +- python +- ruby + +See the [lambda-builder](https://github.com/dokku/lambda-builder#how-does-it-work) documentation for more information on how specific languages are detected. + +### Customizing the build environment + +The `lambda` builder plugin delegates all build logic to [lambda-builder](https://github.com/dokku/lambda-builder), including language detection and build/runtime environment specification. The lambda-builder tool supports a `lambda.yml` file format for customizing how apps are built. Please see the readme for lambda-builder for more information on different options. + +### Changing the `lambda.yml` location + +When deploying a monorepo, it may be desirable to specify the specific path of the `lambda.yml` file to use for a given app. This can be done via the `builder-lambda:set` command. If a value is specified and that file does not exist in the app's build directory, then the build will fail. + +```shell +dokku builder-lambda:set node-js-app lambdayml-path lambda2.yml +``` + +The default value may be set by passing an empty value for the option: + +```shell +dokku builder-lambda:set node-js-app lambdayml-path +``` + +The `lambdayml-path` property can also be set globally. The global default is `lambda.yml`, and the global value is used when no app-specific value is set. + +```shell +dokku builder-lambda:set --global lambdayml-path lambda2.yml +``` + +The default value may be set by passing an empty value for the option. + +```shell +dokku builder-lambda:set --global lambdayml-path +``` + +### Displaying builder-lambda reports for an app + +You can get a report about the app's storage status using the `builder-lambda:report` command: + +```shell +dokku builder-lambda-json:report +``` + +``` +=====> node-js-app builder-lambda information + Builder-lambda computed lambdayml path: lambda2.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: lambda2.yml +=====> python-sample builder-lambda information + Builder-lambda computed lambdayml path: lambda.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: +=====> ruby-sample builder-lambda information + Builder-lambda computed lambdayml path: lambda.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: +``` + +You can run the command for a specific app also. + +```shell +dokku builder-lambda:report node-js-app +``` + +``` +=====> node-js-app builder-lambda information + Builder-lambda computed lambdayml path: lambda2.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: lambda2.yml +``` + +You can pass flags which will output only the value of the specific information you want. For example: + +```shell +dokku builder-lambda:report node-js-app --builder-lambda-lambdayml-path +``` + +``` +Dockerfile2 +``` diff --git a/docs/template.html b/docs/template.html index b07b4de40..06ded964f 100644 --- a/docs/template.html +++ b/docs/template.html @@ -147,6 +147,7 @@ Cloud Native Buildpacks Herokuish Buildpacks Dockerfile Deployment + Lambda Builder Null Builder Deployment Methods From 5ee0619888bbdea08fced96789a96841e6c60750 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 5 Jul 2022 15:24:48 -0400 Subject: [PATCH 12/20] fix: install the lambda-builder package in ci and development --- Makefile | 11 +++++++++-- tests/ci/setup.sh | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f6074a87d..301b7ce47 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ DOKKU_VERSION ?= master DOCKER_IMAGE_LABELER_VERSION ?= 0.5.0 HEROKUISH_VERSION ?= 0.5.37 +LAMBDA_BUILDER_VERSION ?= 0.3.0 NETRC_VERSION ?= 0.6.0 PLUGN_VERSION ?= 0.12.0 PROCFILE_VERSION ?= 0.15.0 SIGIL_VERSION ?= 0.9.0 SSHCOMMAND_VERSION ?= 0.16.0 DOCKER_IMAGE_LABELER_URL ?= https://github.com/dokku/docker-image-labeler/releases/download/v${DOCKER_IMAGE_LABELER_VERSION}/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_linux_amd64.tgz +LAMBDA_BUILDER_URL ?= https://github.com/dokku/lambda-builder/releases/download/v${LAMBDA_BUILDER_VERSION}/lambda-builder_${LAMBDA_BUILDER_VERSION}_linux_amd64.tgz NETRC_URL ?= https://github.com/dokku/netrc/releases/download/v${NETRC_VERSION}/netrc_${NETRC_VERSION}_linux_amd64.tgz PLUGN_URL ?= https://github.com/dokku/plugn/releases/download/v${PLUGN_VERSION}/plugn_${PLUGN_VERSION}_linux_amd64.tgz PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_amd64.tgz @@ -36,7 +38,7 @@ endif include common.mk -.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler netrc sshcommand procfile-util plugn docker aufs stack count vagrant-acl-add vagrant-dokku go-build +.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler lambda-builder netrc sshcommand procfile-util plugn docker aufs stack count vagrant-acl-add vagrant-dokku go-build include tests.mk include package.mk @@ -130,7 +132,7 @@ plugin-dependencies: plugn procfile-util plugins: plugn procfile-util docker sudo -E dokku plugin:install --core -dependencies: apt-update docker-image-labeler netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq parallel +dependencies: apt-update docker-image-labeler lambda-builder netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq parallel $(MAKE) -e stack apt-update: @@ -156,6 +158,11 @@ docker-image-labeler: tar xzf /tmp/docker-image-labeler_latest.tgz -C /usr/local/bin mv /usr/local/bin/docker-image-labeler-amd64 /usr/local/bin/docker-image-labeler +lambda-builder: + wget -qO /tmp/lambda-builder_latest.tgz ${LAMBDA_BUILDER_URL} + tar xzf /tmp/lambda-builder_latest.tgz -C /usr/local/bin + mv /usr/local/bin/lambda-builder-amd64 /usr/local/bin/lambda-builder + netrc: wget -qO /tmp/netrc_latest.tgz ${NETRC_URL} tar xzf /tmp/netrc_latest.tgz -C /usr/local/bin diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 8ac196f3e..89b3f0c94 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -20,6 +20,12 @@ install_dependencies() { curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/herokuish_${HEROKUISH_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}" fi + LAMBDA_BUILDER_VERSION=$(grep LAMBDA_BUILDER_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) + LAMBDA_BUILDER_PACKAGE_NAME="lambda-builder_${LAMBDA_BUILDER_VERSION}_amd64.deb" + if [[ ! -f "$ROOT_DIR/build/${LAMBDA_BUILDER_PACKAGE_NAME}" ]]; then + curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/lambda-builder_${LAMBDA_BUILDER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${LAMBDA_BUILDER_PACKAGE_NAME}" + fi + NETRC_VERSION=$(grep NETRC_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) NETRC_PACKAGE_NAME="netrc_${NETRC_VERSION}_amd64.deb" if [[ ! -f "$ROOT_DIR/build/${NETRC_PACKAGE_NAME}" ]]; then @@ -59,6 +65,7 @@ install_dependencies() { sudo dpkg -i \ "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$LAMBDA_BUILDER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$NETRC_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" \ From d91957b969ccf25fcb77d3240462bf8169d0e769 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 30 Jul 2022 22:48:13 -0400 Subject: [PATCH 13/20] fix: use correct path for source call --- plugins/builder-lambda/core-post-extract | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/builder-lambda/core-post-extract b/plugins/builder-lambda/core-post-extract index 3dc631426..a7a4b6a39 100755 --- a/plugins/builder-lambda/core-post-extract +++ b/plugins/builder-lambda/core-post-extract @@ -1,6 +1,6 @@ #!/usr/bin/env bash source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -source "$PLUGIN_AVAILABLE_PATH/builder-pack/internal-functions" +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" set -eo pipefail [[ $DOKKU_TRACE ]] && set -x From 86d7b6b649707022d082d6304347e29076d24718 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 31 Jul 2022 00:35:18 -0400 Subject: [PATCH 14/20] docs: fix path for alternative lamba.yml --- docs/deployment/builders/lambda.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/builders/lambda.md b/docs/deployment/builders/lambda.md index b2161d2b4..afc7c82ff 100644 --- a/docs/deployment/builders/lambda.md +++ b/docs/deployment/builders/lambda.md @@ -103,5 +103,5 @@ dokku builder-lambda:report node-js-app --builder-lambda-lambdayml-path ``` ``` -Dockerfile2 +lambda2.yml ``` From 30e279b6b90d52837fde8630b84bdde1ad693185 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 9 Aug 2022 02:10:46 -0400 Subject: [PATCH 15/20] fix: suppress labeling output for the dockerfile builder --- plugins/builder-dockerfile/builder-release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/builder-dockerfile/builder-release b/plugins/builder-dockerfile/builder-release index cf3527ffd..bfb623533 100755 --- a/plugins/builder-dockerfile/builder-release +++ b/plugins/builder-dockerfile/builder-release @@ -20,7 +20,7 @@ trigger-builder-dockerfile-builder-release() { trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN INT TERM EXIT local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") - if ! "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-dockerfile/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t "$IMAGE" "$TMP_WORK_DIR"; then + if ! suppress_output "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-dockerfile/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t "$IMAGE" "$TMP_WORK_DIR"; then dokku_log_warn "Failure injecting docker labels on image" return 1 fi From a71e4219be78457c2b5a2e453e34a8b0bac344c2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 9 Aug 2022 02:11:32 -0400 Subject: [PATCH 16/20] fix: set the correct image source type based on label We also have a special carveout for herokuish as folks may deploy built images with it. --- plugins/common/functions | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/plugins/common/functions b/plugins/common/functions index 826524ab4..e2abf8bbf 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -330,6 +330,29 @@ get_running_image_tag() { echo "$IMAGE_TAG" } +get_image_builder_type() { + declare desc="returns the image builder_type" + declare IMAGE="$1" APP="$2" + + if [[ -z "$IMAGE" ]]; then + echo "dockerfile" + return + fi + + BUILDER_TYPE="$("$DOCKER_BIN" image inspect --format '{{ index .Config.Labels "com.dokku.builder-type" }}' "$IMAGE")" + if [[ -n "$BUILDER_TYPE" ]]; then + echo "$BUILDER_TYPE" + return + fi + + if is_image_herokuish_based "$IMAGE" "$APP"; then + echo "herokuish" + return + fi + + echo "dockerfile" +} + is_image_cnb_based() { declare desc="returns true if app image is based on cnb" declare IMAGE="$1" @@ -662,10 +685,8 @@ release_and_deploy() { local DOKKU_DOCKERFILE_PORTS if verify_image "$IMAGE"; then - if is_image_herokuish_based "$IMAGE" "$APP"; then - local IMAGE_SOURCE_TYPE="herokuish" - else - local IMAGE_SOURCE_TYPE="dockerfile" + IMAGE_SOURCE_TYPE="$(get_image_builder_type "$IMAGE" "$APP")" + if [[ "$IMAGE_SOURCE_TYPE" == "dockerfile" ]]; then DOKKU_DOCKERFILE_PORTS=$(config_get "$APP" DOKKU_DOCKERFILE_PORTS || true) if [[ -z "$DOKKU_DOCKERFILE_PORTS" ]]; then local DOCKER_IMAGE_PORTS=$(get_exposed_ports_from_image "$IMAGE") From 2959fdd15407d902e6b7f9688d0270a7f3c208d8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 9 Aug 2022 02:11:48 -0400 Subject: [PATCH 17/20] fix: create the data directory for an app as necessary --- plugins/builder-lambda/builder-build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build index a396e577f..eb13cd21b 100755 --- a/plugins/builder-lambda/builder-build +++ b/plugins/builder-lambda/builder-build @@ -28,7 +28,8 @@ trigger-builder-lambda-builder-build() { fi local GIT_REV="$(plugn trigger git-revision "$APP")" - pusd "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" >/dev/null + mkdir -p "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" + pushd "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" >/dev/null rm -f -- *.zip popd &>/dev/null || pushd "/tmp" >/dev/null cp "$SOURCECODE_WORK_DIR/lambda.zip" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.zip" From 65b9c6aa3e9ebc4540f70fb559c21ba93b66bfbf Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 9 Aug 2022 02:12:27 -0400 Subject: [PATCH 18/20] fix: delete the data dir for the app after the app is deleted --- plugins/builder-lambda/post-delete | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/builder-lambda/post-delete b/plugins/builder-lambda/post-delete index ce2d88aa7..bf296f95b 100755 --- a/plugins/builder-lambda/post-delete +++ b/plugins/builder-lambda/post-delete @@ -9,6 +9,7 @@ trigger-builder-lambda-post-delete() { declare APP="$1" fn-plugin-property-destroy "builder-lambda" "$APP" + rm -rf "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" } trigger-builder-lambda-post-delete "$@" From 7cc84dcdbe9c03f8b113faa5df3e41c982d60014 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 9 Aug 2022 02:13:06 -0400 Subject: [PATCH 19/20] tests: add unit test for lambda builder --- tests/apps/lambda-python/function.py | 6 + tests/apps/lambda-python/requirements.txt | 1 + tests/unit/builder-lambda.bats | 154 ++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 tests/apps/lambda-python/function.py create mode 100644 tests/apps/lambda-python/requirements.txt create mode 100644 tests/unit/builder-lambda.bats diff --git a/tests/apps/lambda-python/function.py b/tests/apps/lambda-python/function.py new file mode 100644 index 000000000..2c5a5dcf5 --- /dev/null +++ b/tests/apps/lambda-python/function.py @@ -0,0 +1,6 @@ +import requests + + +def handler(event, context): + print("Hello World!") + return "Hello World!" diff --git a/tests/apps/lambda-python/requirements.txt b/tests/apps/lambda-python/requirements.txt new file mode 100644 index 000000000..9cdfca0c2 --- /dev/null +++ b/tests/apps/lambda-python/requirements.txt @@ -0,0 +1 @@ +requests==2.27.1 \ No newline at end of file diff --git a/tests/unit/builder-lambda.bats b/tests/unit/builder-lambda.bats new file mode 100644 index 000000000..fe69ae51a --- /dev/null +++ b/tests/unit/builder-lambda.bats @@ -0,0 +1,154 @@ +#!/usr/bin/env bats + +load test_helper + +setup_file() { + install_pack +} + +setup() { + create_app +} + +teardown() { + destroy_app +} + +@test "(builder-lambda:set)" { + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected lambda" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app lambda-python dokku@dokku.me:$TEST_APP inject_lambda_yml + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success + + run /bin/bash -c "dokku builder-lambda:set $TEST_APP lambdayml-path nonexistent.yml" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success + + run /bin/bash -c "dokku builder-lambda:set $TEST_APP lambdayml-path lambda2.yml" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success + + run /bin/bash -c "dokku builder-lambda:set $TEST_APP lambdayml-path" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success +} + +inject_lambda_yml() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + echo "-----> Injecting lambda.yml" + echo "---" >>"$APP_REPO_DIR/lambda.yml" + + echo "-----> Injecting lambda.yml" + echo "---" >>"$APP_REPO_DIR/lambda.yml" + + echo "-----> Injecting lambda2.yml" + echo "---" >>"$APP_REPO_DIR/lambda2.yml" + echo "build_image: mlupin/docker-lambda:python3.9-build" >>"$APP_REPO_DIR/lambda2.yml" + echo "run_image: mlupin/docker-lambda:python3.9" >>"$APP_REPO_DIR/lambda2.yml" +} From 6caf4e6039e026d57e9014b76f06eb726b756b64 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Tue, 9 Aug 2022 02:34:04 -0400 Subject: [PATCH 20/20] fix: use newer lambda-builder --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 301b7ce47..cd792b016 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ DOKKU_VERSION ?= master DOCKER_IMAGE_LABELER_VERSION ?= 0.5.0 HEROKUISH_VERSION ?= 0.5.37 -LAMBDA_BUILDER_VERSION ?= 0.3.0 +LAMBDA_BUILDER_VERSION ?= 0.4.0 NETRC_VERSION ?= 0.6.0 PLUGN_VERSION ?= 0.12.0 PROCFILE_VERSION ?= 0.15.0