From 5c1a1baf08a428b646b43d6139c8403689b74e5d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 1 Mar 2021 04:45:37 -0500 Subject: [PATCH] feat: implement git:from-archive The previous tar support lacked the ability to track changes between tarball deploys. Critically, it also failed to be handled correctly when there was _also_ a git deployment done on the app, resulting in odd deployment states depending on the angles of the moon and the sun in the sky. Rather than try to "fix" this through some hokey mechanism, importing the tar file contents into the git repository is preferred, as then the user can refer to the repository for commit history. Additionally, we add support for non-tar files (tar.gz and zip), enabling deployments from systems that do not create tar files, such as Github (their tarball url is a tar.gz file). Finally, this deprecates the tar plugin, and sets it to be removed in the next minor release (in addition to the tags plugin). Closes #3458 Closes #4207 --- debian/control | 2 +- docs/appendices/0.24.0-migration-guide.md | 3 +- docs/deployment/methods/git.md | 37 +++++++++- docs/deployment/methods/tar.md | 2 + plugins/git/git-from-directory | 1 + plugins/git/help-functions | 3 +- plugins/git/internal-functions | 87 ++++++++++++++++++++++- plugins/git/subcommands/from-archive | 6 ++ rpm.mk | 1 + tests/unit/git_5.bats | 75 +++++++++++++++++++ 10 files changed, 210 insertions(+), 7 deletions(-) create mode 100755 plugins/git/subcommands/from-archive create mode 100644 tests/unit/git_5.bats diff --git a/debian/control b/debian/control index 07e87fb74..17d09be3e 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.23.9 Section: web Priority: optional Architecture: amd64 -Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, netrc, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq +Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, netrc, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq, unzip Recommends: herokuish (>= 0.3.4), parallel, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/docs/appendices/0.24.0-migration-guide.md b/docs/appendices/0.24.0-migration-guide.md index 3a538eed4..73234f806 100644 --- a/docs/appendices/0.24.0-migration-guide.md +++ b/docs/appendices/0.24.0-migration-guide.md @@ -8,4 +8,5 @@ ## Deprecations - The 1.0.0 release of Dokku will no longer select buildpack builders over dockerfile builders if both builders match. Instead, Dokku will choose the first builder that responds to the `builder-detect` trigger. Users that wish to use a specific builder may set a builder using the `builder:set` command, which will force Dokku to use the specified builder regardless of what might be auto-detected. -- The `tags` plugin is deprecated in favor of the [`git:from-image`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-a-docker-image) command. It will be removed in a future release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-image`. +- The `tags` plugin is deprecated in favor of the [`git:from-image`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-a-docker-image) command. It will be removed in the next minor release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-image`. +- The `tar` plugin is deprecated in favor of the [`git:from-archive`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-an-archive-file) command. It will be removed in the next minor release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-archive`. diff --git a/docs/deployment/methods/git.md b/docs/deployment/methods/git.md index ecae5b9f7..fa601c883 100644 --- a/docs/deployment/methods/git.md +++ b/docs/deployment/methods/git.md @@ -5,7 +5,8 @@ ``` git:allow-host # Adds a host to known_hosts git:auth [ ] # Configures netrc authentication for a given git server -git:from-image [--build-dir DIRECTORY] [ ] # Updates a app's git repository with a given docker image +git:from-archive [--archive-type ARCHIVE_TYPE] [ ] # Updates an app's git repository with a given archive file +git:from-image [--build-dir DIRECTORY] [ ] # Updates an app's git repository with a given docker image git:sync [--build] [] # Clone or fetch an app from remote git repo git:initialize # Initialize a git repository for an app git:public-key # Outputs the dokku public deploy key @@ -131,7 +132,7 @@ In the above example, Dokku will build the app as if the repository contained _o FROM dokku/node-js-getting-started:latest ``` -Triggering a build with the same arguments multiple times will result in Dokku exiting `0` early as there is no build to perform. +Triggering a build with the same arguments multiple times will result in Dokku exiting `0` early as there will be no changes detected. The `git:from-image` command can optionally take a git `user.name` and `user.email` argument (in that order) to customize the author. If the arguments are left empty, they will fallback to `Dokku` and `automated@dokku.sh`, respectively. @@ -147,6 +148,38 @@ dokku git:from-image --build-dir path/to/build node-js-app dokku/node-js-getting See the [dockerfile documentation](/docs/deployment/methods/dockerfiles.md) to learn about the different ways to configure Dockerfile-based deploys. +### Initializing an app repository from an archive file + +> New as of 0.24.0 + +A Dokku app repository can be initialized or updated from the contents of an archive file via the `git:from-archive` command. This is an excellent way of tracking changes when deploying pre-built binary archives, such as java jars or go binaries. This can also be useful when deploying directly from a Github repository at a specific commit. + +```shell +dokku git:from-archive node-js-app https://github.com/dokku/smoke-test-app/releases/download/2.0.0/smoke-test-app.tar +``` + +In the above example, Dokku will build the app as if the repository contained the extracted contents of the specified archive file. + +Triggering a build with the same archive file multiple times will result in Dokku exiting `0` early as there will be no changes detected. + +The `git:from-archive` command can optionally take a git `user.name` and `user.email` argument (in that order) to customize the author. If the arguments are left empty, they will fallback to `Dokku` and `automated@dokku.sh`, respectively. + +```shell +dokku git:from-archive node-js-app https://github.com/dokku/smoke-test-app/releases/download/2.0.0/smoke-test-app.tar "Camila" "camila@example.com" +``` + +The default archive type is always set to `.tar`. To use a different archive type, specify the `--archive-type` flag. Failure to do so will result in a failure to extract the archive. + +```shell +dokku git:from-archive --archive-type zip node-js-app https://github.com/dokku/smoke-test-app/archive/2.0.0.zip "Camila" "camila@example.com" +``` + +Finally, if the archive url is specified as `--`, the archive will be fetched from stdin. + +```shell +curl -sSL https://github.com/dokku/smoke-test-app/releases/download/2.0.0/smoke-test-app.tar | dokku git:from-archive node-js-app -- +``` + ### Initializing an app repository from a remote repository > New as of 0.23.0 diff --git a/docs/deployment/methods/tar.md b/docs/deployment/methods/tar.md index 4cfb3baa4..5a873c1f5 100644 --- a/docs/deployment/methods/tar.md +++ b/docs/deployment/methods/tar.md @@ -1,5 +1,7 @@ # Tarball Deployment +> Warning: As of 0.24.0, this functionality is deprecated in favor of the [`git:from-archive`](/docs/deployment/methods/git.md#initializing-an-app-repository-from-an-archive-file) command. It will be removed in a future release, and is considered unmaintained. Users are highly encouraged to switch their workflows to `git:from-arhive`. +> > New as of 0.4.0 ``` diff --git a/plugins/git/git-from-directory b/plugins/git/git-from-directory index a1504257d..27ddde79b 100755 --- a/plugins/git/git-from-directory +++ b/plugins/git/git-from-directory @@ -51,6 +51,7 @@ trigger-git-git-from-directory() { fn-git-fetch "$APP" "$TMP_WORK_DIR" "$REV" fi + rm -rf "$TMP_WORK_DIR" "$TMP_WORK_DIR_2" >/dev/null git_receive_app "$APP" "$REV" } diff --git a/plugins/git/help-functions b/plugins/git/help-functions index 3fc05a844..42f64c255 100755 --- a/plugins/git/help-functions +++ b/plugins/git/help-functions @@ -29,7 +29,8 @@ fn-help-content() { cat <, Adds a host to known_hosts git:auth [ ], Configures netrc authentication for a given git server - git:from-image [ ], Updates a app's git repository with a given docker image + git:from-archive [ ], Updates an app's git repository with a given archive file + git:from-image [ ], Updates an app's git repository with a given docker image git:sync [--build] [], Clone or fetch an app from remote git repo git:initialize , Initialize a git repository for an app git:public-key, Outputs the dokku public deploy key diff --git a/plugins/git/internal-functions b/plugins/git/internal-functions index 3a7cfec45..c3b05c7ae 100755 --- a/plugins/git/internal-functions +++ b/plugins/git/internal-functions @@ -15,6 +15,89 @@ cmd-git-allow-host() { dokku_log_info1 "$HOST added to known hosts" } +cmd-git-from-archive() { + declare desc="updates an app's git repository with a given archive file" + local cmd="git:from-archive" + [[ "$1" == "$cmd" ]] && shift 1 + declare APP ARCHIVE_URL USER_NAME USER_EMAIL + local ARCHIVE_TYPE="tar" + + ARGS=() + skip=false + for arg in "$@"; do + if [[ "$arg" == "--archive-type" ]]; then + skip=true + continue + fi + + if [[ "$skip" == "true" ]]; then + ARCHIVE_TYPE="$arg" + skip=false + continue + fi + + ARGS+=("$arg") + done + + APP="${ARGS[0]}" + ARCHIVE_URL="${ARGS[1]}" + USER_NAME="${ARGS[2]}" + USER_EMAIL="${ARGS[3]}" + + verify_app_name "$APP" + [[ -z "$ARCHIVE_URL" ]] && dokku_log_fail "Please specify an archive url or -- to fetch the archive from stdin" + + local VALID_ARCHIVE_TYPES=("tar" "tar.gz" "zip") + if ! fn-in-array "$ARCHIVE_TYPE" "${ARCHIVE_TYPE[@]}"; then + dokku_log_fail "Invalid archive type specified, valid archive types include: tar, tar.gz, zip" + fi + + local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") + local TMP_WORK_DIR_2=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") + local TMP_WORK_DIR_3=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") + trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' '$TMP_WORK_DIR_3' >/dev/null" RETURN INT TERM EXIT + + if [[ "$ARCHIVE_URL" == "--" ]]; then + dokku_log_info1 "Fetching $ARCHIVE_TYPE file from stdin" + tee "$TMP_WORK_DIR_2/src.$ARCHIVE_TYPE" | wc -c + else + dokku_log_info1 "Downloading $ARCHIVE_TYPE file from $ARCHIVE_URL" + curl -# -L "$ARCHIVE_URL" -o "$TMP_WORK_DIR_2/src.$ARCHIVE_TYPE" + fi + + dokku_log_info1 "Generating build context" + if [[ "$ARCHIVE_TYPE" == "tar" ]]; then + local COMMON_PREFIX=$(tar -tf "$TMP_WORK_DIR_2/src.tar" | sed -e 'N;s/^\(.*\).*\n\1.*$/\1\n\1/;D') + local BOGUS_PARTS=$(echo "$COMMON_PREFIX " | awk 'BEGIN{FS="/"} {print NF-1}') + dokku_log_verbose "Striping $BOGUS_PARTS worth of directories from tarball" + tar -x -C "$TMP_WORK_DIR_3" -f "$TMP_WORK_DIR_2/src.tar" --strip-components="$BOGUS_PARTS" + elif [[ "$ARCHIVE_TYPE" == "tar.gz" ]]; then + dokku_log_verbose "Extracting gzipped tarball" + tar -x -C "$TMP_WORK_DIR_3" -f "$TMP_WORK_DIR_2/src.tar.gz" -z + elif [[ "$ARCHIVE_TYPE" == "zip" ]]; then + dokku_log_verbose "Extracting zipball" + unzip -d "$TMP_WORK_DIR_3" "$TMP_WORK_DIR_2/src.zip" + fi + + chmod -R u+r "$TMP_WORK_DIR_3" + + # drop any top-level folder components that resulted from the folder extraction + if [[ "$(find "$TMP_WORK_DIR_3" -maxdepth 1 -printf %y)" == "dd" ]]; then + dokku_log_verbose "Stripping top-level archive folder components" + local subpath="$(find "$TMP_WORK_DIR_3" -mindepth 1 -maxdepth 1 -type d)" + pushd "$subpath" >/dev/null + find . -maxdepth 1 -exec mv {} "$TMP_WORK_DIR" \; + popd &>/dev/null || pushd "/tmp" >/dev/null + else + dokku_log_verbose "Moving unarchived files and folders into place" + pushd "$TMP_WORK_DIR_3" >/dev/null + find . -maxdepth 1 -exec mv {} "$TMP_WORK_DIR" \; + popd &>/dev/null || pushd "/tmp" >/dev/null + fi + + plugn trigger git-from-directory "$APP" "$TMP_WORK_DIR" "$USER_NAME" "$USER_EMAIL" +} + cmd-git-auth() { declare desc="configures netrc authentication for a given git server" local cmd="git:auth" @@ -37,7 +120,7 @@ cmd-git-auth() { } cmd-git-from-image() { - declare desc="updates a app's git repository with a given docker image" + declare desc="updates an app's git repository with a given docker image" local cmd="git:from-image" [[ "$1" == "$cmd" ]] && shift 1 declare APP DOCKER_IMAGE USER_NAME USER_EMAIL @@ -69,7 +152,7 @@ cmd-git-from-image() { [[ -z "$DOCKER_IMAGE" ]] && dokku_log_fail "Please specify a docker image" local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX") - trap "rm -rf '$TMP_WORK_DIR' '$TMP_WORK_DIR_2' >/dev/null" RETURN INT TERM EXIT + trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN INT TERM EXIT dokku_log_info1 "Generating build context" if [[ -n "$BUILD_DIR" ]]; then diff --git a/plugins/git/subcommands/from-archive b/plugins/git/subcommands/from-archive new file mode 100755 index 000000000..cc3880637 --- /dev/null +++ b/plugins/git/subcommands/from-archive @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/git/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-git-from-archive "$@" diff --git a/rpm.mk b/rpm.mk index 4b7bf6b10..5d1c636da 100644 --- a/rpm.mk +++ b/rpm.mk @@ -47,6 +47,7 @@ endif --depends '/usr/bin/python3' \ --depends 'sshcommand >= 0.11.0' \ --depends 'sudo' \ + --depends 'unzip' \ --after-install rpm/dokku.postinst \ --url "https://github.com/$(DOKKU_REPO_NAME)" \ --description $(DOKKU_DESCRIPTION) \ diff --git a/tests/unit/git_5.bats b/tests/unit/git_5.bats new file mode 100644 index 000000000..016eab882 --- /dev/null +++ b/tests/unit/git_5.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup + create_app + touch /home/dokku/.ssh/known_hosts + chown dokku:dokku /home/dokku/.ssh/known_hosts +} + +teardown() { + rm -f /home/dokku/.ssh/id_rsa.pub || true + destroy_app + global_teardown +} + +@test "(git) git:from-archive [missing]" { + run /bin/bash -c "dokku git:from-archive $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_failure +} + +@test "(git) git:from-archive [invalid archive type]" { + run /bin/bash -c "dokku git:from-archive --archive-type tarball $TEST_APP http://example.com/src.tar" + echo "output: $output" + echo "status: $status" + assert_failure +} + +@test "(git) git:from-archive [tar]" { + run /bin/bash -c "dokku git:from-archive $TEST_APP https://github.com/dokku/smoke-test-app/releases/download/2.0.0/smoke-test-app.tar" + echo "output: $output" + echo "status: $status" + assert_success + assert_http_success http://${TEST_APP}.dokku.me +} + +@test "(git) git:from-archive [tar.gz]" { + run /bin/bash -c "dokku git:from-archive --archive-type tar.gz $TEST_APP https://github.com/dokku/smoke-test-app/archive/2.0.0.tar.gz" + echo "output: $output" + echo "status: $status" + assert_success + assert_http_success http://${TEST_APP}.dokku.me +} + +@test "(git) git:from-archive [zip]" { + run /bin/bash -c "dokku git:from-archive --archive-type zip $TEST_APP https://github.com/dokku/smoke-test-app/archive/2.0.0.zip" + echo "output: $output" + echo "status: $status" + assert_success + assert_http_success http://${TEST_APP}.dokku.me +} + +@test "(git) git:from-archive [stdin-file]" { + run /bin/bash -c "curl -sL https://github.com/dokku/smoke-test-app/releases/download/2.0.0/smoke-test-app.tar -o /tmp/smoke-test-app.tar" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "cat /tmp/smoke-test-app.tar | dokku git:from-archive $TEST_APP --" + echo "output: $output" + echo "status: $status" + assert_success + assert_http_success http://${TEST_APP}.dokku.me +} + +@test "(git) git:from-archive [stdin-curl]" { + run /bin/bash -c "curl -sL https://github.com/dokku/smoke-test-app/releases/download/2.0.0/smoke-test-app.tar | dokku git:from-archive $TEST_APP --" + echo "output: $output" + echo "status: $status" + assert_success + assert_http_success http://${TEST_APP}.dokku.me +}