diff --git a/Makefile b/Makefile index f6074a87d..cd792b016 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ DOKKU_VERSION ?= master DOCKER_IMAGE_LABELER_VERSION ?= 0.5.0 HEROKUISH_VERSION ?= 0.5.37 +LAMBDA_BUILDER_VERSION ?= 0.4.0 NETRC_VERSION ?= 0.6.0 PLUGN_VERSION ?= 0.12.0 PROCFILE_VERSION ?= 0.15.0 SIGIL_VERSION ?= 0.9.0 SSHCOMMAND_VERSION ?= 0.16.0 DOCKER_IMAGE_LABELER_URL ?= https://github.com/dokku/docker-image-labeler/releases/download/v${DOCKER_IMAGE_LABELER_VERSION}/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_linux_amd64.tgz +LAMBDA_BUILDER_URL ?= https://github.com/dokku/lambda-builder/releases/download/v${LAMBDA_BUILDER_VERSION}/lambda-builder_${LAMBDA_BUILDER_VERSION}_linux_amd64.tgz NETRC_URL ?= https://github.com/dokku/netrc/releases/download/v${NETRC_VERSION}/netrc_${NETRC_VERSION}_linux_amd64.tgz PLUGN_URL ?= https://github.com/dokku/plugn/releases/download/v${PLUGN_VERSION}/plugn_${PLUGN_VERSION}_linux_amd64.tgz PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_amd64.tgz @@ -36,7 +38,7 @@ endif include common.mk -.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler netrc sshcommand procfile-util plugn docker aufs stack count vagrant-acl-add vagrant-dokku go-build +.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler lambda-builder netrc sshcommand procfile-util plugn docker aufs stack count vagrant-acl-add vagrant-dokku go-build include tests.mk include package.mk @@ -130,7 +132,7 @@ plugin-dependencies: plugn procfile-util plugins: plugn procfile-util docker sudo -E dokku plugin:install --core -dependencies: apt-update docker-image-labeler netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq parallel +dependencies: apt-update docker-image-labeler lambda-builder netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq parallel $(MAKE) -e stack apt-update: @@ -156,6 +158,11 @@ docker-image-labeler: tar xzf /tmp/docker-image-labeler_latest.tgz -C /usr/local/bin mv /usr/local/bin/docker-image-labeler-amd64 /usr/local/bin/docker-image-labeler +lambda-builder: + wget -qO /tmp/lambda-builder_latest.tgz ${LAMBDA_BUILDER_URL} + tar xzf /tmp/lambda-builder_latest.tgz -C /usr/local/bin + mv /usr/local/bin/lambda-builder-amd64 /usr/local/bin/lambda-builder + netrc: wget -qO /tmp/netrc_latest.tgz ${NETRC_URL} tar xzf /tmp/netrc_latest.tgz -C /usr/local/bin diff --git a/debian/control b/debian/control index b872c85c5..6f21771cf 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.27.10 Section: web Priority: optional Architecture: amd64 -Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, netrc, software-properties-common, parallel, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip +Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip Recommends: herokuish (>= 0.3.4), bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/docs/deployment/builders/builder-management.md b/docs/deployment/builders/builder-management.md index e15e239b8..4c383751b 100644 --- a/docs/deployment/builders/builder-management.md +++ b/docs/deployment/builders/builder-management.md @@ -17,6 +17,7 @@ Dokku supports the following built-in builders: - `builder-dockerfile`: Builds apps using a `Dockerfile` via `docker build`. See the [dockerfile builder documentation](/docs/deployment/builders/dockerfiles.md) for more information on how this builder functions. - `builder-herokuish`: Builds apps with Heroku's v2a Buildpack specification via `gliderlabs/herokuish`. See the [herokuish builder documentation](/docs/deployment/builders/herokuish-buildpacks.md) for more information on how this builder functions. +- `builder-lambda`: Builds AWS Lambda functions in an environment simulating AWS Lambda runtimes via [lambda-builder](https://github.com/dokku/lambda-builder). See the [lambda builder documentation](/docs/deployment/builders/lambda.md) for more information on how this builder functions. - `builder-null`: Does nothing during the build phase. See the [null builder documentation](/docs/deployment/builders/null.md) for more information on how this builder functions. - `builder-pack`: Builds apps with Cloud Native Buildpacks via the `pack-cli` tool. See the [cloud native buildpacks builder documentation](/docs/deployment/builders/cloud-native-buildpacks.md) for more information on how this builder functions. diff --git a/docs/deployment/builders/lambda.md b/docs/deployment/builders/lambda.md new file mode 100644 index 000000000..afc7c82ff --- /dev/null +++ b/docs/deployment/builders/lambda.md @@ -0,0 +1,107 @@ +# Lambda Builder + +> New as of 0.28.0 + +The `lambda` builder builds AWS Lambda functions in an environment simulating AWS Lambda runtimes via [lambda-builder](https://github.com/dokku/lambda-builder). Apps built via this builder can run natively in Dokku and may also have their artifacts scheduled to Lambda via an appropriate scheduler. + +## Usage + +### Detection + +This builder will be auto-detected in the following case: + +- A `lambda.yml` exists in the root of the app repository. + +The builder may also be selected via the `builder:set` command + +```shell +dokku builder:set node-js-app selected lambda +``` + +### Supported languages + +The `lambda` builder plugin supports the following AWS runtime languages on Amazon Linux 2: + +- dotnet +- go (builder is based on AL1) +- nodejs +- python +- ruby + +See the [lambda-builder](https://github.com/dokku/lambda-builder#how-does-it-work) documentation for more information on how specific languages are detected. + +### Customizing the build environment + +The `lambda` builder plugin delegates all build logic to [lambda-builder](https://github.com/dokku/lambda-builder), including language detection and build/runtime environment specification. The lambda-builder tool supports a `lambda.yml` file format for customizing how apps are built. Please see the readme for lambda-builder for more information on different options. + +### Changing the `lambda.yml` location + +When deploying a monorepo, it may be desirable to specify the specific path of the `lambda.yml` file to use for a given app. This can be done via the `builder-lambda:set` command. If a value is specified and that file does not exist in the app's build directory, then the build will fail. + +```shell +dokku builder-lambda:set node-js-app lambdayml-path lambda2.yml +``` + +The default value may be set by passing an empty value for the option: + +```shell +dokku builder-lambda:set node-js-app lambdayml-path +``` + +The `lambdayml-path` property can also be set globally. The global default is `lambda.yml`, and the global value is used when no app-specific value is set. + +```shell +dokku builder-lambda:set --global lambdayml-path lambda2.yml +``` + +The default value may be set by passing an empty value for the option. + +```shell +dokku builder-lambda:set --global lambdayml-path +``` + +### Displaying builder-lambda reports for an app + +You can get a report about the app's storage status using the `builder-lambda:report` command: + +```shell +dokku builder-lambda-json:report +``` + +``` +=====> node-js-app builder-lambda information + Builder-lambda computed lambdayml path: lambda2.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: lambda2.yml +=====> python-sample builder-lambda information + Builder-lambda computed lambdayml path: lambda.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: +=====> ruby-sample builder-lambda information + Builder-lambda computed lambdayml path: lambda.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: +``` + +You can run the command for a specific app also. + +```shell +dokku builder-lambda:report node-js-app +``` + +``` +=====> node-js-app builder-lambda information + Builder-lambda computed lambdayml path: lambda2.yml + Builder-lambda global lambdayml path: lambda.yml + Builder-lambda lambdayml path: lambda2.yml +``` + +You can pass flags which will output only the value of the specific information you want. For example: + +```shell +dokku builder-lambda:report node-js-app --builder-lambda-lambdayml-path +``` + +``` +lambda2.yml +``` diff --git a/docs/template.html b/docs/template.html index b07b4de40..06ded964f 100644 --- a/docs/template.html +++ b/docs/template.html @@ -147,6 +147,7 @@ Cloud Native Buildpacks Herokuish Buildpacks Dockerfile Deployment + Lambda Builder Null Builder Deployment Methods diff --git a/plugins/builder-dockerfile/builder-release b/plugins/builder-dockerfile/builder-release index cf3527ffd..bfb623533 100755 --- a/plugins/builder-dockerfile/builder-release +++ b/plugins/builder-dockerfile/builder-release @@ -20,7 +20,7 @@ trigger-builder-dockerfile-builder-release() { trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN INT TERM EXIT local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") - if ! "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-dockerfile/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t "$IMAGE" "$TMP_WORK_DIR"; then + if ! suppress_output "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-dockerfile/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t "$IMAGE" "$TMP_WORK_DIR"; then dokku_log_warn "Failure injecting docker labels on image" return 1 fi diff --git a/plugins/builder-lambda/builder-build b/plugins/builder-lambda/builder-build new file mode 100755 index 000000000..eb13cd21b --- /dev/null +++ b/plugins/builder-lambda/builder-build @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-builder-build() { + declare desc="builder-lambda builder-build plugin trigger" + declare trigger="builder-build" + declare BUILDER_TYPE="$1" APP="$2" SOURCECODE_WORK_DIR="$3" + + if [[ "$BUILDER_TYPE" != "lambda" ]]; then + return + fi + + dokku_log_info1 "Building $APP from lambda" + + local IMAGE=$(get_app_image_name "$APP") + + pushd "$SOURCECODE_WORK_DIR" &>/dev/null + + plugn trigger pre-build-lambda "$APP" + + # shellcheck disable=SC2086 + lambda-builder build --generate-image --write-procfile --image-env=DOCKER_LAMBDA_STAY_OPEN=1 --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.image-stage=build --label=com.dokku.builder-type=lambda "--label=com.dokku.app-name=$APP" $DOKKU_GLOBAL_BUILD_ARGS --port 5000 --tag "$IMAGE" --working-directory "$SOURCECODE_WORK_DIR" + if [[ ! -f "$SOURCECODE_WORK_DIR/lambda.zip" ]]; then + dokku_log_warn "Compressed lambda.zip not detected, failed to build lambda function" + return 1 + fi + + local GIT_REV="$(plugn trigger git-revision "$APP")" + mkdir -p "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" + pushd "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" >/dev/null + rm -f -- *.zip + popd &>/dev/null || pushd "/tmp" >/dev/null + cp "$SOURCECODE_WORK_DIR/lambda.zip" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.zip" + if [[ -f "$SOURCECODE_WORK_DIR/Procfile" ]]; then + cp "$SOURCECODE_WORK_DIR/Procfile" "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP/$GIT_REV.Procfile" + fi + + plugn trigger post-build-lambda "$APP" +} + +trigger-builder-lambda-builder-build "$@" diff --git a/plugins/builder-lambda/builder-detect b/plugins/builder-lambda/builder-detect new file mode 100755 index 000000000..930c41ef1 --- /dev/null +++ b/plugins/builder-lambda/builder-detect @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/config/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-builder-detect() { + declare desc="builder-lambda builder-detect plugin trigger" + declare trigger="builder-detect" + declare APP="$1" SOURCECODE_WORK_DIR="$2" + local ARCHITECTURE="$(dpkg --print-architecture 2>/dev/null || true)" + + if [[ "$ARCHITECTURE" == "armhf" ]]; then + return + fi + + if [[ -f "$SOURCECODE_WORK_DIR/lambda.yml" ]]; then + echo "lambda" + return + fi +} + +trigger-builder-lambda-builder-detect "$@" diff --git a/plugins/builder-lambda/builder-release b/plugins/builder-lambda/builder-release new file mode 100755 index 000000000..087cb8007 --- /dev/null +++ b/plugins/builder-lambda/builder-release @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-builder-release() { + declare desc="builder-lambda builder-release plugin trigger" + declare trigger="builder-release" + declare BUILDER_TYPE="$1" APP="$2" IMAGE_TAG="$3" + + if [[ "$BUILDER_TYPE" != "lambda" ]]; then + return + fi + + local DOCKER_BUILD_LABEL_ARGS="--label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=com.dokku.app-name=$APP --label=com.dokku.image-stage=release --label=dokku" + + plugn trigger pre-release-lambda "$APP" "$IMAGE_TAG" + + TMP_WORK_DIR="$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")" + trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN INT TERM EXIT + + local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") + if ! suppress_output "$DOCKER_BIN" image build "${DOCKER_BUILD_LABEL_ARGS[@]}" $DOKKU_GLOBAL_BUILD_ARGS -f "$PLUGIN_AVAILABLE_PATH/builder-lambda/dockerfiles/builder-release.Dockerfile" --build-arg APP_IMAGE="$IMAGE" -t "$IMAGE" "$TMP_WORK_DIR"; then + dokku_log_warn "Failure injecting docker labels on image" + return 1 + fi + + plugn trigger post-release-lambda "$APP" "$IMAGE_TAG" + plugn trigger post-release-builder "$BUILDER_TYPE" "$APP" "$IMAGE" +} + +trigger-builder-lambda-builder-release "$@" diff --git a/plugins/builder-lambda/commands b/plugins/builder-lambda/commands new file mode 100755 index 000000000..0e355afce --- /dev/null +++ b/plugins/builder-lambda/commands @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/help-functions" + +case "$1" in + help | builder-lambda:help) + cmd-builder-lambda-help "$@" + ;; + + *) + exit "$DOKKU_NOT_IMPLEMENTED_EXIT" + ;; + +esac diff --git a/plugins/builder-lambda/core-post-extract b/plugins/builder-lambda/core-post-extract new file mode 100755 index 000000000..a7a4b6a39 --- /dev/null +++ b/plugins/builder-lambda/core-post-extract @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-core-post-extract() { + declare desc="builder-lambda post-extract plugin trigger" + declare trigger="post-extract" + declare APP="$1" SOURCECODE_WORK_DIR="$2" + local NEW_LAMBDA_YML="$(fn-builder-lambda-computed-lambdayml-path "$APP")" + + pushd "$TMP_WORK_DIR" >/dev/null + + if [[ -z "$NEW_LAMBDA_YML" ]]; then + return + fi + + if [[ ! -f "$NEW_LAMBDA_YML" ]]; then + rm -f lambda.yml + return + fi + + if [[ "$NEW_LAMBDA_YML" != "lambda.yml" ]]; then + mv "$NEW_LAMBDA_YML" lambda.yml + fi + popd &>/dev/null || pushd "/tmp" >/dev/null +} + +trigger-builder-lambda-core-post-extract "$@" diff --git a/plugins/builder-lambda/dockerfiles/builder-release.Dockerfile b/plugins/builder-lambda/dockerfiles/builder-release.Dockerfile new file mode 100644 index 000000000..cca256f21 --- /dev/null +++ b/plugins/builder-lambda/dockerfiles/builder-release.Dockerfile @@ -0,0 +1,2 @@ +ARG APP_IMAGE +FROM $APP_IMAGE diff --git a/plugins/builder-lambda/help-functions b/plugins/builder-lambda/help-functions new file mode 100755 index 000000000..341fe44ac --- /dev/null +++ b/plugins/builder-lambda/help-functions @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-help() { + declare desc="help command" + declare CMD="$1" + local plugin_name="builder-lambda" + local plugin_description="Manage the lambda builder integration for an app" + + if [[ "$CMD" == "${plugin_name}:help" ]]; then + echo -e "Usage: dokku ${plugin_name}[:COMMAND]" + echo '' + echo "$plugin_description" + echo '' + echo 'Additional commands:' + fn-help-content | sort | column -c2 -t -s, + elif [[ $(ps -o command= $PPID) == *"--all"* ]]; then + fn-help-content + else + cat <] [], Displays a builder-lambda report for one or more apps + builder-lambda:set (), Set or clear a builder-lambda property for an app +help_content +} diff --git a/plugins/builder-lambda/install b/plugins/builder-lambda/install new file mode 100755 index 000000000..66c8aa98d --- /dev/null +++ b/plugins/builder-lambda/install @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-install() { + declare desc="installs the builder-lambda plugin" + declare trigger="install" + + fn-plugin-property-setup "builder-lambda" + + mkdir -p "${DOKKU_LIB_ROOT}/data/builder-lambda" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/builder-lambda" +} + +trigger-builder-lambda-install "$@" diff --git a/plugins/builder-lambda/internal-functions b/plugins/builder-lambda/internal-functions new file mode 100755 index 000000000..d9341ae88 --- /dev/null +++ b/plugins/builder-lambda/internal-functions @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-report() { + declare desc="displays a builder-lambda report for one or more apps" + declare cmd="builder-lambda:report" + [[ "$1" == "$cmd" ]] && shift 1 + declare APP="$1" INFO_FLAG="$2" + + if [[ -n "$APP" ]] && [[ "$APP" == --* ]]; then + INFO_FLAG="$APP" + APP="" + fi + + if [[ -z "$APP" ]] && [[ -z "$INFO_FLAG" ]]; then + INFO_FLAG="true" + fi + + if [[ -z "$APP" ]]; then + for app in $(dokku_apps); do + cmd-builder-lambda-report-single "$app" "$INFO_FLAG" | tee || true + done + else + cmd-builder-lambda-report-single "$APP" "$INFO_FLAG" + fi +} + +cmd-builder-lambda-report-single() { + declare APP="$1" INFO_FLAG="$2" + if [[ "$INFO_FLAG" == "true" ]]; then + INFO_FLAG="" + fi + verify_app_name "$APP" + local flag_map=( + "--builder-lambda-computed-lambdayml-path: $(fn-builder-lambda-computed-lambdayml-path "$APP")" + "--builder-lambda-global-lambdayml-path: $(fn-builder-lambda-global-lambdayml-path "$APP")" + "--builder-lambda-lambdayml-path: $(fn-builder-lambda-lambdayml-path "$APP")" + ) + + if [[ -z "$INFO_FLAG" ]]; then + dokku_log_info2_quiet "${APP} builder-lambda information" + for flag in "${flag_map[@]}"; do + key="$(echo "${flag#--}" | cut -f1 -d' ' | tr - ' ')" + dokku_log_verbose "$(printf "%-30s %-25s" "${key^}" "${flag#*: }")" + done + else + local match=false + local value_exists=false + for flag in "${flag_map[@]}"; do + valid_flags="${valid_flags} $(echo "$flag" | cut -d':' -f1)" + if [[ "$flag" == "${INFO_FLAG}:"* ]]; then + value=${flag#*: } + size="${#value}" + if [[ "$size" -ne 0 ]]; then + echo "$value" && match=true && value_exists=true + else + match=true + fi + fi + done + [[ "$match" == "true" ]] || dokku_log_fail "Invalid flag passed, valid flags:${valid_flags}" + [[ "$value_exists" == "true" ]] || dokku_log_fail "not deployed" + fi +} + +fn-builder-lambda-computed-lambdayml-path() { + declare APP="$1" + + file="$(fn-builder-lambda-lambdayml-path "$APP")" + if [[ "$file" == "" ]]; then + file="$(fn-builder-lambda-global-lambdayml-path "$APP")" + fi + + echo "$file" +} + +fn-builder-lambda-global-lambdayml-path() { + declare APP="$1" + + fn-plugin-property-get "builder-lambda" "--global" "lambdayml-path" "lambda.yml" +} + +fn-builder-lambda-lambdayml-path() { + declare APP="$1" + + fn-plugin-property-get "builder-lambda" "$APP" "lambdayml-path" "" +} diff --git a/plugins/builder-lambda/plugin.toml b/plugins/builder-lambda/plugin.toml new file mode 100644 index 000000000..223a9dd01 --- /dev/null +++ b/plugins/builder-lambda/plugin.toml @@ -0,0 +1,4 @@ +[plugin] +description = "dokku core builder-lambda plugin" +version = "0.27.0" +[plugin.config] diff --git a/plugins/builder-lambda/post-app-clone-setup b/plugins/builder-lambda/post-app-clone-setup new file mode 100755 index 000000000..086a05200 --- /dev/null +++ b/plugins/builder-lambda/post-app-clone-setup @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-post-app-clone-setup() { + declare desc="sets up properties for new app" + declare trigger="post-app-clone-setup" + declare OLD_APP="$1" NEW_APP="$2" + + fn-plugin-property-clone "builder-lambda" "$OLD_APP" "$NEW_APP" +} + +trigger-builder-lambda-post-app-clone-setup "$@" diff --git a/plugins/builder-lambda/post-app-rename-setup b/plugins/builder-lambda/post-app-rename-setup new file mode 100755 index 000000000..c947a84b3 --- /dev/null +++ b/plugins/builder-lambda/post-app-rename-setup @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-post-app-rename-setup() { + declare desc="updates settings for new app" + declare trigger="post-app-rename-setup" + declare OLD_APP="$1" NEW_APP="$2" + + fn-plugin-property-clone "builder-lambda" "$OLD_APP" "$NEW_APP" + fn-plugin-property-destroy "builder-lambda" "$OLD_APP" +} + +trigger-builder-lambda-post-app-rename-setup "$@" diff --git a/plugins/builder-lambda/post-delete b/plugins/builder-lambda/post-delete new file mode 100755 index 000000000..bf296f95b --- /dev/null +++ b/plugins/builder-lambda/post-delete @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-lambda-post-delete() { + declare desc="destroys the builder-lambda properties for a given app" + declare trigger="post-delete" + declare APP="$1" + + fn-plugin-property-destroy "builder-lambda" "$APP" + rm -rf "${DOKKU_LIB_ROOT}/data/builder-lambda/$APP" +} + +trigger-builder-lambda-post-delete "$@" diff --git a/plugins/builder-lambda/report b/plugins/builder-lambda/report new file mode 100755 index 000000000..3271b3684 --- /dev/null +++ b/plugins/builder-lambda/report @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-report-single "$@" diff --git a/plugins/builder-lambda/subcommands/default b/plugins/builder-lambda/subcommands/default new file mode 100755 index 000000000..124c65ef9 --- /dev/null +++ b/plugins/builder-lambda/subcommands/default @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/help-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-help "builder-lambda:help" diff --git a/plugins/builder-lambda/subcommands/report b/plugins/builder-lambda/subcommands/report new file mode 100755 index 000000000..9f56cd32f --- /dev/null +++ b/plugins/builder-lambda/subcommands/report @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/builder-lambda/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-report "$@" diff --git a/plugins/builder-lambda/subcommands/set b/plugins/builder-lambda/subcommands/set new file mode 100755 index 000000000..fcbe16954 --- /dev/null +++ b/plugins/builder-lambda/subcommands/set @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-builder-lambda-set() { + declare desc="set or clear a builder-lambda property for an app" + declare cmd="builder-lambda:set" + [[ "$1" == "$cmd" ]] && shift 1 + declare APP="$1" KEY="$2" VALUE="$3" + local VALID_KEYS=("lambdayml-path") + [[ "$APP" == "--global" ]] || verify_app_name "$APP" + + [[ -z "$KEY" ]] && dokku_log_fail "No key specified" + + if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then + dokku_log_fail "Invalid key specified, valid keys include: lambdayml-path" + fi + + if [[ -n "$VALUE" ]]; then + dokku_log_info2_quiet "Setting ${KEY} to ${VALUE}" + fn-plugin-property-write "builder-lambda" "$APP" "$KEY" "$VALUE" + else + dokku_log_info2_quiet "Unsetting ${KEY}" + fn-plugin-property-delete "builder-lambda" "$APP" "$KEY" + fi +} + +cmd-builder-lambda-set "$@" diff --git a/plugins/builder-lambda/templates/samconfig.toml.sigil b/plugins/builder-lambda/templates/samconfig.toml.sigil new file mode 100644 index 000000000..9af08445b --- /dev/null +++ b/plugins/builder-lambda/templates/samconfig.toml.sigil @@ -0,0 +1,13 @@ +version = 0.1 + +[default] +[default.deploy] +[default.deploy.parameters] +stack_name = "{{ $.APP }}-lambda" +s3_bucket = "{{ $.S3_BUCKET }}" +s3_prefix = "{{ $.APP }}" +region = "{{ $.AWS_REGION }}" +confirm_changeset = true +fail_on_empty_changeset = false +capabilities = "CAPABILITY_IAM" +image_repositories = [] diff --git a/plugins/builder-lambda/templates/template.yaml.sigil b/plugins/builder-lambda/templates/template.yaml.sigil new file mode 100644 index 000000000..dab06e7bc --- /dev/null +++ b/plugins/builder-lambda/templates/template.yaml.sigil @@ -0,0 +1,25 @@ +--- +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: > + {{ $.APP }} + +# todo: iterate over every process type and create a resource entry +Resources: + {{ $.APP}}{{ $.PROCESS_TYPE | capitalize}}: + Type: AWS::Serverless::Function + Properties: + Architectures: + - x86_64 + CodeUri: s3://{{ $.S3_BUCKET }}/{{ $.APP }}/{{ $.GIT_REV }} + FunctionName: {{ $.APP | lower }}{{ $.PROCESS_TYPE | lower }} + Handler: {{ $.HANDLER }} + Runtime: {{ $.RUNTIME }} + Role: {{ $.IAM_ROLE }} + Timeout: 3 + Environment: + Variables: + {{ range $key, $value := . }} + {{ $key }}: | + {{ $value }} + {{ end }} diff --git a/plugins/common/functions b/plugins/common/functions index 826524ab4..e2abf8bbf 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -330,6 +330,29 @@ get_running_image_tag() { echo "$IMAGE_TAG" } +get_image_builder_type() { + declare desc="returns the image builder_type" + declare IMAGE="$1" APP="$2" + + if [[ -z "$IMAGE" ]]; then + echo "dockerfile" + return + fi + + BUILDER_TYPE="$("$DOCKER_BIN" image inspect --format '{{ index .Config.Labels "com.dokku.builder-type" }}' "$IMAGE")" + if [[ -n "$BUILDER_TYPE" ]]; then + echo "$BUILDER_TYPE" + return + fi + + if is_image_herokuish_based "$IMAGE" "$APP"; then + echo "herokuish" + return + fi + + echo "dockerfile" +} + is_image_cnb_based() { declare desc="returns true if app image is based on cnb" declare IMAGE="$1" @@ -662,10 +685,8 @@ release_and_deploy() { local DOKKU_DOCKERFILE_PORTS if verify_image "$IMAGE"; then - if is_image_herokuish_based "$IMAGE" "$APP"; then - local IMAGE_SOURCE_TYPE="herokuish" - else - local IMAGE_SOURCE_TYPE="dockerfile" + IMAGE_SOURCE_TYPE="$(get_image_builder_type "$IMAGE" "$APP")" + if [[ "$IMAGE_SOURCE_TYPE" == "dockerfile" ]]; then DOKKU_DOCKERFILE_PORTS=$(config_get "$APP" DOKKU_DOCKERFILE_PORTS || true) if [[ -z "$DOKKU_DOCKERFILE_PORTS" ]]; then local DOCKER_IMAGE_PORTS=$(get_exposed_ports_from_image "$IMAGE") diff --git a/tests/apps/lambda-python/function.py b/tests/apps/lambda-python/function.py new file mode 100644 index 000000000..2c5a5dcf5 --- /dev/null +++ b/tests/apps/lambda-python/function.py @@ -0,0 +1,6 @@ +import requests + + +def handler(event, context): + print("Hello World!") + return "Hello World!" diff --git a/tests/apps/lambda-python/requirements.txt b/tests/apps/lambda-python/requirements.txt new file mode 100644 index 000000000..9cdfca0c2 --- /dev/null +++ b/tests/apps/lambda-python/requirements.txt @@ -0,0 +1 @@ +requests==2.27.1 \ No newline at end of file diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 8ac196f3e..89b3f0c94 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -20,6 +20,12 @@ install_dependencies() { curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/herokuish_${HEROKUISH_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}" fi + LAMBDA_BUILDER_VERSION=$(grep LAMBDA_BUILDER_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) + LAMBDA_BUILDER_PACKAGE_NAME="lambda-builder_${LAMBDA_BUILDER_VERSION}_amd64.deb" + if [[ ! -f "$ROOT_DIR/build/${LAMBDA_BUILDER_PACKAGE_NAME}" ]]; then + curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/lambda-builder_${LAMBDA_BUILDER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${LAMBDA_BUILDER_PACKAGE_NAME}" + fi + NETRC_VERSION=$(grep NETRC_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) NETRC_PACKAGE_NAME="netrc_${NETRC_VERSION}_amd64.deb" if [[ ! -f "$ROOT_DIR/build/${NETRC_PACKAGE_NAME}" ]]; then @@ -59,6 +65,7 @@ install_dependencies() { sudo dpkg -i \ "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$LAMBDA_BUILDER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$NETRC_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" \ diff --git a/tests/unit/builder-lambda.bats b/tests/unit/builder-lambda.bats new file mode 100644 index 000000000..fe69ae51a --- /dev/null +++ b/tests/unit/builder-lambda.bats @@ -0,0 +1,154 @@ +#!/usr/bin/env bats + +load test_helper + +setup_file() { + install_pack +} + +setup() { + create_app +} + +teardown() { + destroy_app +} + +@test "(builder-lambda:set)" { + run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku builder:set $TEST_APP selected lambda" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app lambda-python dokku@dokku.me:$TEST_APP inject_lambda_yml + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success + + run /bin/bash -c "dokku builder-lambda:set $TEST_APP lambdayml-path nonexistent.yml" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success + + run /bin/bash -c "dokku builder-lambda:set $TEST_APP lambdayml-path lambda2.yml" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success + + run /bin/bash -c "dokku builder-lambda:set $TEST_APP lambdayml-path" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains 'Building app with image mlupin/docker-lambda:python3.9-build' + assert_output_contains 'Installing dependencies via pip' + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" 0 + assert_success + + run /bin/bash -c "curl -d {} --silent --write-out '%{http_code}\n' $(dokku url $TEST_APP)/2015-03-31/functions/function.handler/invocations | grep 200" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_output_contains "Hello World" + assert_success +} + +inject_lambda_yml() { + local APP="$1" + local APP_REPO_DIR="$2" + [[ -z "$APP" ]] && local APP="$TEST_APP" + echo "-----> Injecting lambda.yml" + echo "---" >>"$APP_REPO_DIR/lambda.yml" + + echo "-----> Injecting lambda.yml" + echo "---" >>"$APP_REPO_DIR/lambda.yml" + + echo "-----> Injecting lambda2.yml" + echo "---" >>"$APP_REPO_DIR/lambda2.yml" + echo "build_image: mlupin/docker-lambda:python3.9-build" >>"$APP_REPO_DIR/lambda2.yml" + echo "run_image: mlupin/docker-lambda:python3.9" >>"$APP_REPO_DIR/lambda2.yml" +}