From 9306011f70e125af718f2c3488d94cc630010fa2 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 23 Jan 2021 23:00:22 -0500 Subject: [PATCH] fix: do not retag images unnecessarily This change minimizes the work needed to be done when tagging images. It edits the image manifest directly only when necessary, allowing restarts of an app to avoid having an extra layer. This also additionally allows to deploy images with ONBUILD directives without running the ONBUILD directives. Lastly, users building docker images that run Dokku will need to use a new sudoer wrapper for the `docker-image-labeler` binary to work correctly. A reference version has been placed in the `docker` skeleton directory. Closes #3931 Refs #4226 --- Makefile | 16 +++++-- debian/control | 2 +- docker/etc/sudoers.d/dokku-docker | 2 +- docker/usr/local/bin/docker-image-labeler | 20 ++++++++ docs/appendices/0.23.0-migration-guide.md | 1 + plugins/builder-cnb/builder-release | 2 +- plugins/builder-dockerfile/builder-release | 6 +-- plugins/builder-herokuish/builder-release | 2 +- rpm.mk | 1 + tests/ci/setup.sh | 21 +++++--- tests/unit/tags.bats | 56 +++++++++++++++++++++- 11 files changed, 107 insertions(+), 22 deletions(-) create mode 100755 docker/usr/local/bin/docker-image-labeler diff --git a/Makefile b/Makefile index 7e09ada7b..aca54e5f5 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,16 @@ DOKKU_VERSION ?= master +DOCKER_IMAGE_LABELER_VERSION ?= 0.2.0 HEROKUISH_VERSION ?= 0.5.19 -PROCFILE_VERSION ?= 0.11.0 PLUGN_VERSION ?= 0.6.1 +PROCFILE_VERSION ?= 0.11.0 SIGIL_VERSION ?= 0.6.0 SSHCOMMAND_VERSION ?= 0.12.0 -SSHCOMMAND_URL ?= https://github.com/dokku/sshcommand/releases/download/v${SSHCOMMAND_VERSION}/sshcommand_${SSHCOMMAND_VERSION}_linux_x86_64.tgz -PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_x86_64.tgz +DOCKER_IMAGE_LABELER_URL ?= https://github.com/dokku/docker-image-labeler/releases/download/v${DOCKER_IMAGE_LABELER_VERSION}/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_linux_x86_64.tgz PLUGN_URL ?= https://github.com/dokku/plugn/releases/download/v${PLUGN_VERSION}/plugn_${PLUGN_VERSION}_linux_x86_64.tgz +PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_x86_64.tgz SIGIL_URL ?= https://github.com/gliderlabs/sigil/releases/download/v${SIGIL_VERSION}/sigil_${SIGIL_VERSION}_Linux_x86_64.tgz +SSHCOMMAND_URL ?= https://github.com/dokku/sshcommand/releases/download/v${SSHCOMMAND_VERSION}/sshcommand_${SSHCOMMAND_VERSION}_linux_x86_64.tgz STACK_URL ?= https://github.com/gliderlabs/herokuish.git PREBUILT_STACK_URL ?= gliderlabs/herokuish:latest DOKKU_LIB_ROOT ?= /var/lib/dokku @@ -32,7 +34,7 @@ endif include common.mk -.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build +.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build include tests.mk include package.mk @@ -123,7 +125,7 @@ plugin-dependencies: plugn procfile-util plugins: plugn procfile-util docker sudo -E dokku plugin:install --core -dependencies: apt-update sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq +dependencies: apt-update docker-image-labeler sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq $(MAKE) -e stack apt-update: @@ -141,6 +143,10 @@ help2man: man-db: apt-get -qq -y --no-install-recommends install man-db +docker-image-labeler: + wget -qO /tmp/docker-image-labeler_latest.tgz ${DOCKER_IMAGE_LABELER_URL} + tar xzf /tmp/docker-image-labeler_latest.tgz -C /usr/local/bin + sshcommand: wget -qO /tmp/sshcommand_latest.tgz ${SSHCOMMAND_URL} tar xzf /tmp/sshcommand_latest.tgz -C /usr/local/bin diff --git a/debian/control b/debian/control index d3c3e1d4d..c3f51cbfd 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.22.9 Section: web Priority: optional Architecture: amd64 -Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, net-tools, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq +Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.0), net-tools, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq Recommends: herokuish (>= 0.3.4), parallel, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/docker/etc/sudoers.d/dokku-docker b/docker/etc/sudoers.d/dokku-docker index ea099f997..d5677f34f 100644 --- a/docker/etc/sudoers.d/dokku-docker +++ b/docker/etc/sudoers.d/dokku-docker @@ -1 +1 @@ -dokku ALL=NOPASSWD:SETENV:/usr/bin/docker +dokku ALL=NOPASSWD:SETENV:/usr/bin/docker,/usr/bin/docker-image-labeler diff --git a/docker/usr/local/bin/docker-image-labeler b/docker/usr/local/bin/docker-image-labeler new file mode 100755 index 000000000..d7f0a64a2 --- /dev/null +++ b/docker/usr/local/bin/docker-image-labeler @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $TRACE ]] && set -x + +main() { + declare desc="re-runs docker-image-labeler commands as sudo" + local DOCKER_IMAGE_LABELER_BIN="" + if [[ -x "/usr/bin/docker-image-labeler" ]]; then + DOCKER_IMAGE_LABELER_BIN="/usr/bin/docker-image-labeler" + fi + + if [[ -z "$DOCKER_IMAGE_LABELER_BIN" ]]; then + echo "! No docker-image-labeler binary found" 1>&2 + exit 1 + fi + + sudo -E "$DOCKER_IMAGE_LABELER_BIN" "$@" +} + +main "$@" diff --git a/docs/appendices/0.23.0-migration-guide.md b/docs/appendices/0.23.0-migration-guide.md index 453bdd3f8..c6d499fbd 100644 --- a/docs/appendices/0.23.0-migration-guide.md +++ b/docs/appendices/0.23.0-migration-guide.md @@ -3,3 +3,4 @@ ## Changes - The `plugin:list` command no longer outputs the version for the `plugn` binary. +- Users building docker images that run Dokku will need to use a new sudoer wrapper for the `docker-image-labeler` binary to work correctly. A reference version has been placed in the `docker` skeleton directory. This should only impact platform developers, and users of our Docker image will already have the file available. diff --git a/plugins/builder-cnb/builder-release b/plugins/builder-cnb/builder-release index 1591417fd..e3bb97d6b 100755 --- a/plugins/builder-cnb/builder-release +++ b/plugins/builder-cnb/builder-release @@ -15,7 +15,7 @@ trigger-builder-cnb-builder-release() { plugn trigger pre-release-cnb "$APP" "$IMAGE_TAG" local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") - echo "FROM $IMAGE" | suppress_output "$DOCKER_BIN" image build --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku -t "$IMAGE" - + docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" plugn trigger post-release-cnb "$APP" "$IMAGE_TAG" } diff --git a/plugins/builder-dockerfile/builder-release b/plugins/builder-dockerfile/builder-release index 976c27a86..8e92cff53 100755 --- a/plugins/builder-dockerfile/builder-release +++ b/plugins/builder-dockerfile/builder-release @@ -15,11 +15,7 @@ trigger-builder-dockerfile-builder-release() { plugn trigger pre-release-dockerfile "$APP" "$IMAGE_TAG" local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") - if "$DOCKER_BIN" image history --format "{{ json . }}" "$IMAGE" | grep -q "ONBUILD"; then - dokku_log_warn "Image contains one or more ONBUILD directives, skipping label injection" - else - echo "FROM $IMAGE" | suppress_output "$DOCKER_BIN" image build --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku -t "$IMAGE" - - fi + docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" plugn trigger post-release-dockerfile "$APP" "$IMAGE_TAG" } diff --git a/plugins/builder-herokuish/builder-release b/plugins/builder-herokuish/builder-release index 9ea165df1..9609c390f 100755 --- a/plugins/builder-herokuish/builder-release +++ b/plugins/builder-herokuish/builder-release @@ -40,7 +40,7 @@ trigger-builder-herokuish-builder-release() { plugn trigger scheduler-register-retired "$APP" "$CID" fi - echo "FROM $IMAGE" | suppress_output "$DOCKER_BIN" image build --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku -t "$IMAGE" - + docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" plugn trigger post-release-buildpack "$APP" "$IMAGE_TAG" } diff --git a/rpm.mk b/rpm.mk index 5faf98857..8492f4b3f 100644 --- a/rpm.mk +++ b/rpm.mk @@ -35,6 +35,7 @@ endif --depends 'cpio' \ --depends 'curl' \ --depends 'dos2unix' \ + --depends 'docker-image-labeler >= 0.2.0' \ --depends 'git' \ --depends 'gliderlabs-sigil' \ --depends 'jq' \ diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 7e7e5d7c7..aff1e503e 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -7,6 +7,13 @@ install_dependencies() { echo "=====> install_dependencies on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" mkdir -p "$ROOT_DIR/build/" + + DOCKER_IMAGE_LABELER_VERSION=$(grep DOCKER_IMAGE_LABELER_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) + DOCKER_IMAGE_LABELER_PACKAGE_NAME="docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_amd64.deb" + if [[ ! -f "$ROOT_DIR/build/${DOCKER_IMAGE_LABELER_PACKAGE_NAME}" ]]; then + curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${DOCKER_IMAGE_LABELER_PACKAGE_NAME}" + fi + HEROKUISH_VERSION=$(grep HEROKUISH_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) HEROKUISH_PACKAGE_NAME="herokuish_${HEROKUISH_VERSION}_amd64.deb" if [[ ! -f "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}" ]]; then @@ -42,11 +49,13 @@ install_dependencies() { sudo apt-get -qq -y --no-install-recommends install cgroupfs-mount dos2unix jq nginx debconf-utils sudo cp "${ROOT_DIR}/tests/dhparam.pem" /etc/nginx/dhparam.pem - sudo dpkg -i "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ + sudo dpkg -i \ + "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \ - "${ROOT_DIR}/build/$SSHCOMMAND_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" \ "${ROOT_DIR}/build/$SIGIL_PACKAGE_NAME" \ - "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" + "${ROOT_DIR}/build/$SSHCOMMAND_PACKAGE_NAME" } build_dokku() { @@ -149,10 +158,10 @@ setup_circle() { getent passwd | cut -d: -f1 | sort sudo -E mkdir -p /home/dokku/.dokkurc - if [[ -n "$GITHUB_ACTIONS" ]]; then - sudo -E chown dokku:dokku /home/dokku/.dokkurc - else + if [[ -n "$CIRCLECI" ]]; then sudo -E chown dokku:ubuntu /home/dokku/.dokkurc + else + sudo -E chown dokku:dokku /home/dokku/.dokkurc fi sudo -E chmod 775 /home/dokku/.dokkurc # pull node:4-alpine image for testing diff --git a/tests/unit/tags.bats b/tests/unit/tags.bats index 4cdc4d77d..44bd9b9f6 100644 --- a/tests/unit/tags.bats +++ b/tests/unit/tags.bats @@ -95,7 +95,59 @@ teardown() { run /bin/bash -c "dokku tags:deploy $TEST_APP 3.2.13" echo "output: $output" echo "status: $status" - assert_output_contains "Image contains one or more ONBUILD directives, skipping label injection" - assert_output_contains "COPY failed" 0 assert_success } + +@test "(tags) deploy and restart inject no extra layers" { + run /bin/bash -c "docker image pull linuxserver/foldingathome:7.5.1-ls1" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect linuxserver/foldingathome:7.5.1-ls1 | jq -r '.[0].RootFS.Layers | length'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "6" + layer_count="$output" + + run /bin/bash -c "docker image tag linuxserver/foldingathome:7.5.1-ls1 dokku/$TEST_APP:latest" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect dokku/$TEST_APP:latest | jq -r '.[0].RootFS.Layers | length'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "6" + layer_count="$output" + + run /bin/bash -c "dokku tags:deploy $TEST_APP latest" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect dokku/$TEST_APP:latest | jq -r '.[0].RootFS.Layers | length'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$layer_count" + + run /bin/bash -c "dokku ps:restart $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "docker inspect dokku/$TEST_APP:latest | jq -r '.[0].RootFS.Layers | length'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$layer_count" + + run /bin/bash -c "docker inspect linuxserver/foldingathome:7.5.1-ls1 | jq -r '.[0].RootFS.Layers | length'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "$layer_count" +}