fix: do not retag images unnecessarily

This change minimizes the work needed to be done when tagging images. It edits the image manifest directly only when necessary, allowing restarts of an app to avoid having an extra layer. This also additionally allows to deploy images with ONBUILD directives without running the ONBUILD directives. Lastly, users building docker images that run Dokku will need to use a new sudoer wrapper for the `docker-image-labeler` binary to work correctly. A reference version has been placed in the `docker` skeleton directory. Closes #3931 Refs #4226
2021-01-23 23:00:22 -05:00
parent 162abcee4f
commit 9306011f70
11 changed files with 107 additions and 22 deletions
--- a/16
+++ b/16
@@ -1,14 +1,16 @@
 DOKKU_VERSION ?= master

+DOCKER_IMAGE_LABELER_VERSION ?= 0.2.0
 HEROKUISH_VERSION ?= 0.5.19
-PROCFILE_VERSION ?= 0.11.0
 PLUGN_VERSION ?= 0.6.1
+PROCFILE_VERSION ?= 0.11.0
 SIGIL_VERSION ?= 0.6.0
 SSHCOMMAND_VERSION ?= 0.12.0
-SSHCOMMAND_URL ?= https://github.com/dokku/sshcommand/releases/download/v${SSHCOMMAND_VERSION}/sshcommand_${SSHCOMMAND_VERSION}_linux_x86_64.tgz
-PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_x86_64.tgz
+DOCKER_IMAGE_LABELER_URL ?= https://github.com/dokku/docker-image-labeler/releases/download/v${DOCKER_IMAGE_LABELER_VERSION}/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_linux_x86_64.tgz
 PLUGN_URL ?= https://github.com/dokku/plugn/releases/download/v${PLUGN_VERSION}/plugn_${PLUGN_VERSION}_linux_x86_64.tgz
+PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_x86_64.tgz
 SIGIL_URL ?= https://github.com/gliderlabs/sigil/releases/download/v${SIGIL_VERSION}/sigil_${SIGIL_VERSION}_Linux_x86_64.tgz
+SSHCOMMAND_URL ?= https://github.com/dokku/sshcommand/releases/download/v${SSHCOMMAND_VERSION}/sshcommand_${SSHCOMMAND_VERSION}_linux_x86_64.tgz
 STACK_URL ?= https://github.com/gliderlabs/herokuish.git
 PREBUILT_STACK_URL ?= gliderlabs/herokuish:latest
 DOKKU_LIB_ROOT ?= /var/lib/dokku
@@ -32,7 +34,7 @@ endif

 include common.mk

-.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build
+.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build

 include tests.mk
 include package.mk
@@ -123,7 +125,7 @@ plugin-dependencies: plugn procfile-util
 plugins: plugn procfile-util docker
 	sudo -E dokku plugin:install --core

-dependencies: apt-update sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq
+dependencies: apt-update docker-image-labeler sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq
 	$(MAKE) -e stack

 apt-update:
@@ -141,6 +143,10 @@ help2man:
 man-db:
 	apt-get -qq -y --no-install-recommends install man-db

+docker-image-labeler:
+	wget -qO /tmp/docker-image-labeler_latest.tgz ${DOCKER_IMAGE_LABELER_URL}
+	tar xzf /tmp/docker-image-labeler_latest.tgz -C /usr/local/bin
+
 sshcommand:
 	wget -qO /tmp/sshcommand_latest.tgz ${SSHCOMMAND_URL}
 	tar xzf /tmp/sshcommand_latest.tgz -C /usr/local/bin
--- a/debian/control
+++ b/debian/control
@@ -3,7 +3,7 @@ Version: 0.22.9
 Section: web
 Priority: optional
 Architecture: amd64
-Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, net-tools, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq
+Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.0), net-tools, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq
 Recommends: herokuish (>= 0.3.4), parallel, dokku-update, dokku-event-listener
 Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf
 Maintainer: Jose Diaz-Gonzalez <dokku@josediazgonzalez.com>
--- a/docker/etc/sudoers.d/dokku-docker
+++ b/docker/etc/sudoers.d/dokku-docker
@@ -1 +1 @@
-dokku    ALL=NOPASSWD:SETENV:/usr/bin/docker
+dokku    ALL=NOPASSWD:SETENV:/usr/bin/docker,/usr/bin/docker-image-labeler
--- a/docker/usr/local/bin/docker-image-labeler
+++ b/docker/usr/local/bin/docker-image-labeler
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -eo pipefail
+[[ $TRACE ]] && set -x
+
+main() {
+  declare desc="re-runs docker-image-labeler commands as sudo"
+  local DOCKER_IMAGE_LABELER_BIN=""
+  if [[ -x "/usr/bin/docker-image-labeler" ]]; then
+    DOCKER_IMAGE_LABELER_BIN="/usr/bin/docker-image-labeler"
+  fi
+
+  if [[ -z "$DOCKER_IMAGE_LABELER_BIN" ]]; then
+    echo "!   No docker-image-labeler binary found" 1>&2
+    exit 1
+  fi
+
+  sudo -E "$DOCKER_IMAGE_LABELER_BIN" "$@"
+}
+
+main "$@"
--- a/docs/appendices/0.23.0-migration-guide.md
+++ b/docs/appendices/0.23.0-migration-guide.md
@@ -3,3 +3,4 @@
 ## Changes

 - The `plugin:list` command no longer outputs the version for the `plugn` binary.
+- Users building docker images that run Dokku will need to use a new sudoer wrapper for the `docker-image-labeler` binary to work correctly. A reference version has been placed in the `docker` skeleton directory. This should only impact platform developers, and users of our Docker image will already have the file available.
--- a/plugins/builder-cnb/builder-release
+++ b/plugins/builder-cnb/builder-release
@@ -15,7 +15,7 @@ trigger-builder-cnb-builder-release() {
  plugn trigger pre-release-cnb "$APP" "$IMAGE_TAG"

  local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
-  echo "FROM $IMAGE" | suppress_output "$DOCKER_BIN" image build --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku -t "$IMAGE" -
+  docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE"
  plugn trigger post-release-cnb "$APP" "$IMAGE_TAG"
 }

--- a/plugins/builder-dockerfile/builder-release
+++ b/plugins/builder-dockerfile/builder-release
@@ -15,11 +15,7 @@ trigger-builder-dockerfile-builder-release() {
  plugn trigger pre-release-dockerfile "$APP" "$IMAGE_TAG"

  local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
-  if "$DOCKER_BIN" image history --format "{{ json . }}" "$IMAGE" | grep  -q "ONBUILD"; then
-    dokku_log_warn "Image contains one or more ONBUILD directives, skipping label injection"
-  else
-    echo "FROM $IMAGE" | suppress_output "$DOCKER_BIN" image build --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku -t "$IMAGE" -
-  fi
+  docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE"
  plugn trigger post-release-dockerfile "$APP" "$IMAGE_TAG"
 }

--- a/plugins/builder-herokuish/builder-release
+++ b/plugins/builder-herokuish/builder-release
@@ -40,7 +40,7 @@ trigger-builder-herokuish-builder-release() {
    plugn trigger scheduler-register-retired "$APP" "$CID"
  fi

-  echo "FROM $IMAGE" | suppress_output "$DOCKER_BIN" image build --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku -t "$IMAGE" -
+  docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE"
  plugn trigger post-release-buildpack "$APP" "$IMAGE_TAG"
 }

--- a/rpm.mk
+++ b/rpm.mk
@@ -35,6 +35,7 @@ endif
 		--depends 'cpio' \
 		--depends 'curl' \
 		--depends 'dos2unix' \
+		--depends 'docker-image-labeler >= 0.2.0' \
 		--depends 'git' \
 		--depends 'gliderlabs-sigil' \
 		--depends 'jq' \
--- a/tests/ci/setup.sh
+++ b/tests/ci/setup.sh
@@ -7,6 +7,13 @@ install_dependencies() {
  echo "=====> install_dependencies on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX"

  mkdir -p "$ROOT_DIR/build/"
+
+  DOCKER_IMAGE_LABELER_VERSION=$(grep DOCKER_IMAGE_LABELER_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3)
+  DOCKER_IMAGE_LABELER_PACKAGE_NAME="docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_amd64.deb"
+  if [[ ! -f "$ROOT_DIR/build/${DOCKER_IMAGE_LABELER_PACKAGE_NAME}" ]]; then
+    curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${DOCKER_IMAGE_LABELER_PACKAGE_NAME}"
+  fi
+
  HEROKUISH_VERSION=$(grep HEROKUISH_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3)
  HEROKUISH_PACKAGE_NAME="herokuish_${HEROKUISH_VERSION}_amd64.deb"
  if [[ ! -f "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}" ]]; then
@@ -42,11 +49,13 @@ install_dependencies() {
  sudo apt-get -qq -y --no-install-recommends install cgroupfs-mount dos2unix jq nginx debconf-utils
  sudo cp "${ROOT_DIR}/tests/dhparam.pem" /etc/nginx/dhparam.pem

-  sudo dpkg -i "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \
+  sudo dpkg -i \
+    "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \
+    "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \
    "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \
-    "${ROOT_DIR}/build/$SSHCOMMAND_PACKAGE_NAME" \
+    "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" \
    "${ROOT_DIR}/build/$SIGIL_PACKAGE_NAME" \
-    "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME"
+    "${ROOT_DIR}/build/$SSHCOMMAND_PACKAGE_NAME"
 }

 build_dokku() {
@@ -149,10 +158,10 @@ setup_circle() {
  getent passwd | cut -d: -f1 | sort

  sudo -E mkdir -p /home/dokku/.dokkurc
-  if [[ -n "$GITHUB_ACTIONS" ]]; then
-    sudo -E chown dokku:dokku /home/dokku/.dokkurc
-  else
+  if [[ -n "$CIRCLECI" ]]; then
    sudo -E chown dokku:ubuntu /home/dokku/.dokkurc
+  else
+    sudo -E chown dokku:dokku /home/dokku/.dokkurc
  fi
  sudo -E chmod 775 /home/dokku/.dokkurc
  # pull node:4-alpine image for testing
--- a/tests/unit/tags.bats
+++ b/tests/unit/tags.bats
@@ -95,7 +95,59 @@ teardown() {
  run /bin/bash -c "dokku tags:deploy $TEST_APP 3.2.13"
  echo "output: $output"
  echo "status: $status"
-  assert_output_contains "Image contains one or more ONBUILD directives, skipping label injection"
-  assert_output_contains "COPY failed" 0
  assert_success
 }
+
+@test "(tags) deploy and restart inject no extra layers" {
+  run /bin/bash -c "docker image pull linuxserver/foldingathome:7.5.1-ls1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect linuxserver/foldingathome:7.5.1-ls1 | jq -r '.[0].RootFS.Layers | length'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "6"
+  layer_count="$output"
+
+  run /bin/bash -c "docker image tag linuxserver/foldingathome:7.5.1-ls1 dokku/$TEST_APP:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect dokku/$TEST_APP:latest | jq -r '.[0].RootFS.Layers | length'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "6"
+  layer_count="$output"
+
+  run /bin/bash -c "dokku tags:deploy $TEST_APP latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect dokku/$TEST_APP:latest | jq -r '.[0].RootFS.Layers | length'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$layer_count"
+
+  run /bin/bash -c "dokku ps:restart $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "docker inspect dokku/$TEST_APP:latest | jq -r '.[0].RootFS.Layers | length'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$layer_count"
+
+  run /bin/bash -c "docker inspect linuxserver/foldingathome:7.5.1-ls1 | jq -r '.[0].RootFS.Layers | length'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$layer_count"
+}