From 6b42fde02897248a10ac2629cbed8d7688ddac50 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 13 May 2019 02:59:41 -0400 Subject: [PATCH 01/27] fix: handle case where modprobe does not exist This is possible when installing dokku within a docker image. --- debian/postinst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/postinst b/debian/postinst index 789cbb50a..014926a76 100755 --- a/debian/postinst +++ b/debian/postinst @@ -33,7 +33,9 @@ case "$1" in configure) mandb [ ! -x /usr/bin/docker.io ] || ln -sf /usr/bin/docker.io /usr/local/bin/docker - modprobe aufs || echo "WARNING: Restart server to finish installing dokku!" + if [[ -f /sbin/modprobe ]]; then + modprobe aufs || echo "WARNING: Restart server to finish installing dokku!" + fi call-sshcommand create dokku /usr/bin/dokku egrep -i "^docker" /etc/group || groupadd docker From 3eb9c9b3ba58a4eabeae54466b163bc2e5d25906 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 13 May 2019 03:05:50 -0400 Subject: [PATCH 02/27] feat: initial docker installation support Refs #1076 --- Dockerfile | 60 +++++++++++++++++++ docker/etc/cron.d/dokku-retire | 4 ++ docker/etc/logrotate.d/services | 9 +++ docker/etc/my_init.d/10_directories | 36 +++++++++++ .../runit/runsvdir/default/dokku-init/finish | 15 +++++ .../etc/runit/runsvdir/default/dokku-init/run | 14 +++++ docker/etc/runit/runsvdir/default/nginx/run | 7 +++ docker/etc/runit/runsvdir/default/rsyslog/run | 7 +++ docker/etc/sudoers.d/dokku-docker | 1 + docker/usr/local/bin/docker | 19 ++++++ 10 files changed, 172 insertions(+) create mode 100644 Dockerfile create mode 100644 docker/etc/cron.d/dokku-retire create mode 100644 docker/etc/logrotate.d/services create mode 100755 docker/etc/my_init.d/10_directories create mode 100755 docker/etc/runit/runsvdir/default/dokku-init/finish create mode 100755 docker/etc/runit/runsvdir/default/dokku-init/run create mode 100755 docker/etc/runit/runsvdir/default/nginx/run create mode 100755 docker/etc/runit/runsvdir/default/rsyslog/run create mode 100644 docker/etc/sudoers.d/dokku-docker create mode 100755 docker/usr/local/bin/docker diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..934417d80 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +FROM phusion/baseimage:0.11 + +CMD ["/sbin/my_init"] + +RUN apt-get update && apt-get -y upgrade && apt-get -y install \ + nano \ + rsync \ + software-properties-common \ + sudo \ + wget && \ + apt-get purge -y syslog-ng-core && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ARG DOKKU_TAG=0.17.7 +ARG DOKKU_GID=200 +ARG DOKKU_UID=200 +ARG DOKKU_DOCKERFILE=true +ARG DOKKU_WEB_CONFIG=false +ARG DOKKU_HOSTNAME=dokku.invalid +ARG DOKKU_VHOST_ENABLE=false +ARG DOKKU_SKIP_KEY_FILE=true + +RUN addgroup --gid $DOKKU_GID dokku && \ + adduser --uid $DOKKU_UID --gid $DOKKU_GID --disabled-password --gecos "" "dokku" + +RUN export DOKKU_TAG=$DOKKU_TAG \ + DOKKU_DOCKERFILE=$DOKKU_DOCKERFILE \ + DOKKU_WEB_CONFIG=$DOKKU_WEB_CONFIG \ + DOKKU_HOSTNAME=$DOKKU_HOSTNAME \ + DOKKU_VHOST_ENABLE=$DOKKU_VHOST_ENABLE \ + DOKKU_SKIP_KEY_FILE=$DOKKU_SKIP_KEY_FILE && \ + wget https://raw.githubusercontent.com/dokku/dokku/v$DOKKU_TAG/bootstrap.sh && \ + bash bootstrap.sh && \ + apt-get clean && rm -rf bootstrap.sh /var/lib/apt/lists/* /tmp/* /var/tmp/* + +WORKDIR /tmp + +COPY ./docker . + +RUN \ + rsync -a /tmp/ / && \ + rm -rf /tmp/* && \ + rm /etc/runit/runsvdir/default/sshd/down && \ + chown -R dokku:dokku /home/dokku/ && \ + mkdir -p /skel/etc /skel/home /skel/var/lib/dokku /var/log/services && \ + mv /etc/ssh /skel/etc/ssh && \ + mv /home/dokku /skel/home/dokku && \ + mv /var/lib/dokku/config /skel/var/lib/dokku/config && \ + mv /var/lib/dokku/data /skel/var/lib/dokku/data && \ + ln -sf /mount/etc/ssh /etc/ssh && \ + ln -sf /mount/home/dokku /home/dokku && \ + ln -sf /mount/var/lib/dokku/config /var/lib/dokku/config && \ + ln -sf /mount/var/lib/dokku/data /var/lib/dokku/data && \ + mv /etc/my_init.d/00_regen_ssh_host_keys.sh \ + /etc/my_init.d/15_regen_ssh_host_keys && \ + rm /etc/nginx/sites-enabled/default && \ + rm /usr/share/nginx/html/index.html && \ + rm /etc/my_init.d/10_syslog-ng.init && \ + sed -i '/imklog/d' /etc/rsyslog.conf && \ + rm /var/log/btmp /var/log/wtmp /var/log/*log /var/log/apt/* /var/log/dokku/* /var/log/nginx/* diff --git a/docker/etc/cron.d/dokku-retire b/docker/etc/cron.d/dokku-retire new file mode 100644 index 000000000..ff81b231b --- /dev/null +++ b/docker/etc/cron.d/dokku-retire @@ -0,0 +1,4 @@ +PATH=/usr/local/bin:/usr/bin:/bin +SHELL=/bin/bash + +*/5 * * * * root /usr/bin/dokku ps:retire >> /var/log/dokku/retire.log 2>&1 diff --git a/docker/etc/logrotate.d/services b/docker/etc/logrotate.d/services new file mode 100644 index 000000000..f64cc3925 --- /dev/null +++ b/docker/etc/logrotate.d/services @@ -0,0 +1,9 @@ +/var/log/services/* { + daily + missingok + rotate 7 + compress + delaycompress + notifempty + copytruncate +} diff --git a/docker/etc/my_init.d/10_directories b/docker/etc/my_init.d/10_directories new file mode 100755 index 000000000..3b7926bc8 --- /dev/null +++ b/docker/etc/my_init.d/10_directories @@ -0,0 +1,36 @@ +#!/bin/bash +set -eo pipefail + +log-info() { + declare desc="Log info formatter" + echo " $*" 1>&2 +} + +log-fail() { + declare desc="Log fail formatter" + echo "! $*" 1>&2 + exit 1 +} + +main() { + if [[ ! -d /mount ]]; then + log-info "Creating missing /mount" + mkdir -p /mount + fi + + directories=("/etc/ssh" "/home/dokku" "/var/lib/dokku/config" "/var/lib/dokku/data") + for dir in "${directories[@]}"; do + if [[ ! -e "$dir" ]]; then + log-info "Copying ${dir} skel into place" + mkdir -p "/mount${dir}" + rsync -a "/skel${dir}/" "${dir}/" || log-fail "Unable to copy ${dir} skel into place" + fi + + # chown mount directories to "dokku:dokku" + if [[ "$dir" != "/etc/ssh" ]]; then + find "$dir" -print0 | xargs -0 chown -h dokku:dokku + fi + done +} + +main "$@" diff --git a/docker/etc/runit/runsvdir/default/dokku-init/finish b/docker/etc/runit/runsvdir/default/dokku-init/finish new file mode 100755 index 000000000..bb8dd40d8 --- /dev/null +++ b/docker/etc/runit/runsvdir/default/dokku-init/finish @@ -0,0 +1,15 @@ +#!/bin/bash + +exec >> /var/log/services/dokku-restore +exec 2>&1 + +if [[ "$1" -ne 0 ]]; then + echo "Error initializing dokku! Retrying in 30 seconds..." + exec sleep 30 + exit 0 +fi + +echo "Done initializing dokku." +exec sleep infinity + +exit 0 diff --git a/docker/etc/runit/runsvdir/default/dokku-init/run b/docker/etc/runit/runsvdir/default/dokku-init/run new file mode 100755 index 000000000..f056dfe07 --- /dev/null +++ b/docker/etc/runit/runsvdir/default/dokku-init/run @@ -0,0 +1,14 @@ +#!/bin/bash + +exec >> /var/log/services/dokku-init +exec 2>&1 + +echo "Running dokku-init" +cd /tmp || exit 1 + +if [[ -n "$DOKKU_HOSTNAME" ]]; then + echo "dokku dokku/vhost_enable boolean true" | debconf-set-selections + dokku --quiet domains:set-global "$DOKKU_HOSTNAME" +fi + +exec dokku ps:restore diff --git a/docker/etc/runit/runsvdir/default/nginx/run b/docker/etc/runit/runsvdir/default/nginx/run new file mode 100755 index 000000000..192a059b4 --- /dev/null +++ b/docker/etc/runit/runsvdir/default/nginx/run @@ -0,0 +1,7 @@ +#!/bin/bash + +exec >> /var/log/services/nginx +exec 2>&1 + +echo "Running nginx" +exec /usr/sbin/nginx -c /etc/nginx/nginx.conf -g "daemon off;" diff --git a/docker/etc/runit/runsvdir/default/rsyslog/run b/docker/etc/runit/runsvdir/default/rsyslog/run new file mode 100755 index 000000000..942694f45 --- /dev/null +++ b/docker/etc/runit/runsvdir/default/rsyslog/run @@ -0,0 +1,7 @@ +#!/bin/bash + +exec >> /var/log/services/rsyslog +exec 2>&1 + +echo "Running rsyslog" +exec /usr/sbin/rsyslogd -n diff --git a/docker/etc/sudoers.d/dokku-docker b/docker/etc/sudoers.d/dokku-docker new file mode 100644 index 000000000..ea099f997 --- /dev/null +++ b/docker/etc/sudoers.d/dokku-docker @@ -0,0 +1 @@ +dokku ALL=NOPASSWD:SETENV:/usr/bin/docker diff --git a/docker/usr/local/bin/docker b/docker/usr/local/bin/docker new file mode 100755 index 000000000..3efb68413 --- /dev/null +++ b/docker/usr/local/bin/docker @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $TRACE ]] && set -x + +main() { + declare desc="re-runs docker commands as sudo" + local DOCKER_BIN="" + if [[ -x "/usr/bin/docker" ]]; then + DOCKER_BIN="/usr/bin/docker" + fi + + if [[ -z "$DOCKER_BIN" ]]; then + echo "! No docker binary found" 1>&2 + exit 1 + fi + + sudo -E "$DOCKER_BIN" "$@" +} + +main "$@" From 56d8346d65aa2cb22213fcda5a80efe62cdca60b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 13 May 2019 03:22:10 -0400 Subject: [PATCH 03/27] docs: standardizing installation doc titles --- docs/getting-started/install/azure.md | 2 +- docs/getting-started/install/digitalocean.md | 2 +- docs/getting-started/install/dreamhost.md | 2 +- docs/getting-started/install/vagrant.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/getting-started/install/azure.md b/docs/getting-started/install/azure.md index 6d61ddda9..a9beb19c5 100644 --- a/docs/getting-started/install/azure.md +++ b/docs/getting-started/install/azure.md @@ -1,4 +1,4 @@ -# Microsoft Azure +# Microsoft Azure Installation Notes 1. If you don't already have one [generate an SSH key pair](https://help.github.com/articles/generating-ssh-keys/). diff --git a/docs/getting-started/install/digitalocean.md b/docs/getting-started/install/digitalocean.md index 2d9462413..caf2a717b 100644 --- a/docs/getting-started/install/digitalocean.md +++ b/docs/getting-started/install/digitalocean.md @@ -1,4 +1,4 @@ -# DigitalOcean Droplet +# DigitalOcean Droplet Installation Notes [DigitalOcean](https://www.digitalocean.com/products/compute/) offers a pre-installed Dokku image. You can run this image on any sized Droplet, although larger Droplets will allow you to run larger applications. diff --git a/docs/getting-started/install/dreamhost.md b/docs/getting-started/install/dreamhost.md index 0f4d005f2..a664d94f5 100644 --- a/docs/getting-started/install/dreamhost.md +++ b/docs/getting-started/install/dreamhost.md @@ -1,4 +1,4 @@ -# Install Dokku on DreamHost Cloud Server with cloud-init +# DreamHost Cloud Server Installation Notes Cloud-init script can be used to automate installation of Dokku on Dreamhost (or any other OpenStack-compatible cloud with minimal diff --git a/docs/getting-started/install/vagrant.md b/docs/getting-started/install/vagrant.md index 3ea1e9f72..5b8055ed6 100644 --- a/docs/getting-started/install/vagrant.md +++ b/docs/getting-started/install/vagrant.md @@ -1,4 +1,4 @@ -# Install Dokku using Vagrant +# Vagrant Installation Notes 1. Download and install [VirtualBox](https://www.virtualbox.org/wiki/Downloads). From 01f317bbf31dffec69b782aa8f754d56cc8a1b79 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 13 May 2019 03:31:42 -0400 Subject: [PATCH 04/27] feat: initialize plugins on boot --- docker/etc/runit/runsvdir/default/dokku-init/run | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/etc/runit/runsvdir/default/dokku-init/run b/docker/etc/runit/runsvdir/default/dokku-init/run index f056dfe07..43ff2331c 100755 --- a/docker/etc/runit/runsvdir/default/dokku-init/run +++ b/docker/etc/runit/runsvdir/default/dokku-init/run @@ -6,6 +6,8 @@ exec 2>&1 echo "Running dokku-init" cd /tmp || exit 1 +dokku plugin:install + if [[ -n "$DOKKU_HOSTNAME" ]]; then echo "dokku dokku/vhost_enable boolean true" | debconf-set-selections dokku --quiet domains:set-global "$DOKKU_HOSTNAME" From d7bb131b3765de3be71b835c2c29d85d06c244f3 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 13 May 2019 03:39:41 -0400 Subject: [PATCH 05/27] feat: add the ability to install arbitrary plugins on container start --- docker/etc/runit/runsvdir/default/dokku-init/run | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/etc/runit/runsvdir/default/dokku-init/run b/docker/etc/runit/runsvdir/default/dokku-init/run index 43ff2331c..45056db57 100755 --- a/docker/etc/runit/runsvdir/default/dokku-init/run +++ b/docker/etc/runit/runsvdir/default/dokku-init/run @@ -6,7 +6,11 @@ exec 2>&1 echo "Running dokku-init" cd /tmp || exit 1 -dokku plugin:install +if [[ -f /mount/plugin-list ]]; then + while read line; do + dokku plugin:install "$(echo "$line" | awk '{print $2}')" "$(echo "$line" | cut -d':' -f1)" + done Date: Mon, 13 May 2019 03:40:07 -0400 Subject: [PATCH 06/27] docs: add documentation for docker-based installation --- docs/getting-started/install/docker.md | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/getting-started/install/docker.md diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md new file mode 100644 index 000000000..346abcac1 --- /dev/null +++ b/docs/getting-started/install/docker.md @@ -0,0 +1,47 @@ +# Docker Installation Notes + +Pull the dokku/dokku image: + +```shell +docker pull dokku/dokku:v0.16.2 +``` + +Next, run the image. + +```shell +docker run \ + --env DOKKU_HOSTNAME=dokku.me \ + --name dokku \ + --publish 3022:22 \ + --publish 8080:80 \ + --publish 8443:443 \ + --volume /mount:/mount \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + dokku/dokku:v0.16.2 +``` + +Dokku is run with the following options: + +- The global hostname is set to `dokku.me` on boot. +- The container name is dokku. +- SSH port 22 is exposed on the host as 3022. +- HTTP port 80 is exposed on the host as 8000. +- HTTPS port 443 is exposed on the host as 8443. +- Data within the container is stored on the host within the `/mount` directory. +- The docker socket is mounted into container + +Application repositories, plugin config, and plugin data is persisted to disk within the specified host directory for `/mount`. + +To install custom plugins, create a `plugin-list` file in the host's `/mount` directory. The plugins listed herein will be automatically installed by Dokku on container boot. This file should be the following format: + +``` +plugin_name: repository_url +``` + +An example for installing the postgres and redis plugins follows: + +```shell +postgres: https://github.com/dokku/dokku-postgres.git +redis: https://github.com/dokku/dokku-redis.git +``` + From 6fe984fd3c9db98676617c715fb8fd7b50407188 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 13 May 2019 03:40:28 -0400 Subject: [PATCH 07/27] fix: ensure releases take docker-based installation into account when updating version numbers --- contrib/release-dokku | 2 ++ .../{10_directories => 10_dokku_init} | 11 ++++++++++ .../etc/runit/runsvdir/default/dokku-init/run | 20 ------------------- .../{dokku-init => dokku-restore}/finish | 4 ++-- .../runit/runsvdir/default/dokku-restore/run | 9 +++++++++ 5 files changed, 24 insertions(+), 22 deletions(-) rename docker/etc/my_init.d/{10_directories => 10_dokku_init} (68%) delete mode 100755 docker/etc/runit/runsvdir/default/dokku-init/run rename docker/etc/runit/runsvdir/default/{dokku-init => dokku-restore}/finish (60%) create mode 100755 docker/etc/runit/runsvdir/default/dokku-restore/run diff --git a/contrib/release-dokku b/contrib/release-dokku index f88f01a92..a3d2e2443 100755 --- a/contrib/release-dokku +++ b/contrib/release-dokku @@ -215,6 +215,8 @@ fn-repo-update() { fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/getting-started/installation.md fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/template.html fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" README.md + fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" Dockerfile + fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/getting-started/install/docker.md if [[ "$RELEASE" == 'patch' ]]; then fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/assets/versions.json diff --git a/docker/etc/my_init.d/10_directories b/docker/etc/my_init.d/10_dokku_init similarity index 68% rename from docker/etc/my_init.d/10_directories rename to docker/etc/my_init.d/10_dokku_init index 3b7926bc8..3d253fce5 100755 --- a/docker/etc/my_init.d/10_directories +++ b/docker/etc/my_init.d/10_dokku_init @@ -31,6 +31,17 @@ main() { find "$dir" -print0 | xargs -0 chown -h dokku:dokku fi done + + if [[ -f /mount/plugin-list ]]; then + while read line; do + dokku plugin:install "$(echo "$line" | awk '{print $2}')" "$(echo "$line" | cut -d':' -f1)" + done > /var/log/services/dokku-init -exec 2>&1 - -echo "Running dokku-init" -cd /tmp || exit 1 - -if [[ -f /mount/plugin-list ]]; then - while read line; do - dokku plugin:install "$(echo "$line" | awk '{print $2}')" "$(echo "$line" | cut -d':' -f1)" - done > /var/log/services/dokku-restore exec 2>&1 if [[ "$1" -ne 0 ]]; then - echo "Error initializing dokku! Retrying in 30 seconds..." + echo "Error restoring dokku apps! Retrying in 30 seconds..." exec sleep 30 exit 0 fi -echo "Done initializing dokku." +echo "Done restoring dokku apps." exec sleep infinity exit 0 diff --git a/docker/etc/runit/runsvdir/default/dokku-restore/run b/docker/etc/runit/runsvdir/default/dokku-restore/run new file mode 100755 index 000000000..58b368a31 --- /dev/null +++ b/docker/etc/runit/runsvdir/default/dokku-restore/run @@ -0,0 +1,9 @@ +#!/bin/bash + +exec >> /var/log/services/dokku-restore +exec 2>&1 + +echo "Running dokku-restore" +cd /tmp || exit 1 + +exec dokku ps:restore From 9b67ffe9d434c0103a2cbe7f297111433cf271d1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 9 Jun 2019 12:17:32 -0400 Subject: [PATCH 08/27] feat: properly handle dhparam The dhparam file should be specific to each installation, and thus needs to be generated on first run. --- .dockerignore | 1 + Dockerfile | 3 +++ docker/etc/my_init.d/10_dokku_init | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/.dockerignore b/.dockerignore index 84200e4bd..a89678a7f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,4 +10,5 @@ tests !contrib/dokku-update !contrib/bash-completion !contrib/docker +!tests/dhparam.pem vendor diff --git a/Dockerfile b/Dockerfile index 934417d80..51d1b4541 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,9 @@ ARG DOKKU_SKIP_KEY_FILE=true RUN addgroup --gid $DOKKU_GID dokku && \ adduser --uid $DOKKU_UID --gid $DOKKU_GID --disabled-password --gecos "" "dokku" +RUN mkdir /etc/nginx/ +COPY tests/dhparam.pem /etc/nginx/dhparam.pem + RUN export DOKKU_TAG=$DOKKU_TAG \ DOKKU_DOCKERFILE=$DOKKU_DOCKERFILE \ DOKKU_WEB_CONFIG=$DOKKU_WEB_CONFIG \ diff --git a/docker/etc/my_init.d/10_dokku_init b/docker/etc/my_init.d/10_dokku_init index 3d253fce5..1d0dc4a36 100755 --- a/docker/etc/my_init.d/10_dokku_init +++ b/docker/etc/my_init.d/10_dokku_init @@ -42,6 +42,15 @@ main() { echo "dokku dokku/vhost_enable boolean true" | debconf-set-selections dokku --quiet domains:set-global "$DOKKU_HOSTNAME" fi + + if [[ ! -f /mount/etc/nginx/dhparam.pem ]]; then + mkdir -p /mount/etc/nginx + openssl dhparam -out /mount/etc/nginx/dhparam.pem 2048 + fi + + cp -f /mount/etc/nginx/dhparam.pem /etc/nginx/dhparam.pem + chown root:root /etc/nginx/dhparam.pem + chown root:root /etc/nginx/conf.d } main "$@" From ea32544e37cd5da656ffbe751838b48b19f22b60 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 9 Jun 2019 12:17:46 -0400 Subject: [PATCH 09/27] feat: set the debconf value for the global hostname --- docker/etc/my_init.d/10_dokku_init | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/etc/my_init.d/10_dokku_init b/docker/etc/my_init.d/10_dokku_init index 1d0dc4a36..050d075a9 100755 --- a/docker/etc/my_init.d/10_dokku_init +++ b/docker/etc/my_init.d/10_dokku_init @@ -39,6 +39,7 @@ main() { fi if [[ -n "$DOKKU_HOSTNAME" ]]; then + echo "dokku dokku/hostname string $DOKKU_HOSTNAME" | debconf-set-selections echo "dokku dokku/vhost_enable boolean true" | debconf-set-selections dokku --quiet domains:set-global "$DOKKU_HOSTNAME" fi From 1a4af9b3ea133f5618039600f629a68e21a3e547 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 9 Jun 2019 12:34:50 -0400 Subject: [PATCH 10/27] feat: handle builds during the release process --- contrib/release-dokku | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/release-dokku b/contrib/release-dokku index a3d2e2443..e5e1fc951 100755 --- a/contrib/release-dokku +++ b/contrib/release-dokku @@ -360,6 +360,8 @@ main() { if [[ "$RELEASE" != "build" ]]; then fn-publish-package "$IS_RELEASE" "deb" "build/dokku_${NEXT_VERSION}_amd64.deb" || log-fail "Error publishing deb package" fn-publish-package "$IS_RELEASE" "rpm" "build/dokku-${NEXT_VERSION}-1.x86_64.rpm" || log-fail "Error publishing rpm package" + docker build -t "dokku/dokku:$NEXT_VERSION" . + docker push "dokku/dokku:$NEXT_VERSION" fn-repo-push-tags "$IS_RELEASE" fi From c5d5bc9a9c6a41a089b9178fcb64249893cab968 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 9 Jun 2019 12:51:02 -0400 Subject: [PATCH 11/27] chore: minor fixes to docker-based install docs --- contrib/release-dokku | 6 +++--- docs/getting-started/install/docker.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/release-dokku b/contrib/release-dokku index e5e1fc951..3fa7b3890 100755 --- a/contrib/release-dokku +++ b/contrib/release-dokku @@ -206,17 +206,17 @@ fn-repo-update() { fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" contrib/dokku-installer.py fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" debian/control + fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" Dockerfile fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/advanced-usage/plugin-management.md fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/assets/favicons/browserconfig.xml fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/assets/favicons/manifest.json fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/assets/style.css fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/development/release-process.md - fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/home.html + fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/getting-started/install/docker.md fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/getting-started/installation.md + fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/home.html fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/template.html fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" README.md - fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" Dockerfile - fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/getting-started/install/docker.md if [[ "$RELEASE" == 'patch' ]]; then fn-replace-version "$CURRENT_VERSION" "$NEXT_VERSION" docs/assets/versions.json diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md index 346abcac1..524bb548a 100644 --- a/docs/getting-started/install/docker.md +++ b/docs/getting-started/install/docker.md @@ -3,7 +3,7 @@ Pull the dokku/dokku image: ```shell -docker pull dokku/dokku:v0.16.2 +docker pull dokku/dokku:v0.17.6 ``` Next, run the image. @@ -17,7 +17,7 @@ docker run \ --publish 8443:443 \ --volume /mount:/mount \ --volume /var/run/docker.sock:/var/run/docker.sock \ - dokku/dokku:v0.16.2 + dokku/dokku:v0.17.6 ``` Dokku is run with the following options: From a568965d149aae8eea7a8dc07e5a5b19d6701760 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 9 Jun 2019 14:22:17 -0400 Subject: [PATCH 12/27] fix: run shfmt --- docker/etc/my_init.d/10_dokku_init | 2 +- docker/etc/runit/runsvdir/default/dokku-restore/finish | 2 +- docker/etc/runit/runsvdir/default/dokku-restore/run | 2 +- docker/etc/runit/runsvdir/default/nginx/run | 2 +- docker/etc/runit/runsvdir/default/rsyslog/run | 2 +- docker/usr/local/bin/docker | 3 ++- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/etc/my_init.d/10_dokku_init b/docker/etc/my_init.d/10_dokku_init index 050d075a9..a8313c13a 100755 --- a/docker/etc/my_init.d/10_dokku_init +++ b/docker/etc/my_init.d/10_dokku_init @@ -18,7 +18,7 @@ main() { mkdir -p /mount fi - directories=("/etc/ssh" "/home/dokku" "/var/lib/dokku/config" "/var/lib/dokku/data") + directories=("/etc/ssh" "/home/dokku" "/var/lib/dokku/config" "/var/lib/dokku/data") for dir in "${directories[@]}"; do if [[ ! -e "$dir" ]]; then log-info "Copying ${dir} skel into place" diff --git a/docker/etc/runit/runsvdir/default/dokku-restore/finish b/docker/etc/runit/runsvdir/default/dokku-restore/finish index 92f49a93d..ea80c20c0 100755 --- a/docker/etc/runit/runsvdir/default/dokku-restore/finish +++ b/docker/etc/runit/runsvdir/default/dokku-restore/finish @@ -1,6 +1,6 @@ #!/bin/bash -exec >> /var/log/services/dokku-restore +exec >>/var/log/services/dokku-restore exec 2>&1 if [[ "$1" -ne 0 ]]; then diff --git a/docker/etc/runit/runsvdir/default/dokku-restore/run b/docker/etc/runit/runsvdir/default/dokku-restore/run index 58b368a31..4287517ac 100755 --- a/docker/etc/runit/runsvdir/default/dokku-restore/run +++ b/docker/etc/runit/runsvdir/default/dokku-restore/run @@ -1,6 +1,6 @@ #!/bin/bash -exec >> /var/log/services/dokku-restore +exec >>/var/log/services/dokku-restore exec 2>&1 echo "Running dokku-restore" diff --git a/docker/etc/runit/runsvdir/default/nginx/run b/docker/etc/runit/runsvdir/default/nginx/run index 192a059b4..3328faaaf 100755 --- a/docker/etc/runit/runsvdir/default/nginx/run +++ b/docker/etc/runit/runsvdir/default/nginx/run @@ -1,6 +1,6 @@ #!/bin/bash -exec >> /var/log/services/nginx +exec >>/var/log/services/nginx exec 2>&1 echo "Running nginx" diff --git a/docker/etc/runit/runsvdir/default/rsyslog/run b/docker/etc/runit/runsvdir/default/rsyslog/run index 942694f45..9a84a59ed 100755 --- a/docker/etc/runit/runsvdir/default/rsyslog/run +++ b/docker/etc/runit/runsvdir/default/rsyslog/run @@ -1,6 +1,6 @@ #!/bin/bash -exec >> /var/log/services/rsyslog +exec >>/var/log/services/rsyslog exec 2>&1 echo "Running rsyslog" diff --git a/docker/usr/local/bin/docker b/docker/usr/local/bin/docker index 3efb68413..dcebde400 100755 --- a/docker/usr/local/bin/docker +++ b/docker/usr/local/bin/docker @@ -1,5 +1,6 @@ #!/usr/bin/env bash -set -eo pipefail; [[ $TRACE ]] && set -x +set -eo pipefail +[[ $TRACE ]] && set -x main() { declare desc="re-runs docker commands as sudo" From 230a098b7bdbd8cda5ab1e849fe530d5b68b5c1a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 10 Jun 2019 09:32:10 -0400 Subject: [PATCH 13/27] chore: drop unnecessary copy of test dhparam --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51d1b4541..934417d80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,9 +23,6 @@ ARG DOKKU_SKIP_KEY_FILE=true RUN addgroup --gid $DOKKU_GID dokku && \ adduser --uid $DOKKU_UID --gid $DOKKU_GID --disabled-password --gecos "" "dokku" -RUN mkdir /etc/nginx/ -COPY tests/dhparam.pem /etc/nginx/dhparam.pem - RUN export DOKKU_TAG=$DOKKU_TAG \ DOKKU_DOCKERFILE=$DOKKU_DOCKERFILE \ DOKKU_WEB_CONFIG=$DOKKU_WEB_CONFIG \ From 3500dbfa6a039d4c30e5bd7e3dfc7849f6ef1c92 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 10 Jun 2019 16:02:41 -0400 Subject: [PATCH 14/27] refactor: clean up dockerfile to reduce image layers --- Dockerfile | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 934417d80..1709b14e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,36 +2,28 @@ FROM phusion/baseimage:0.11 CMD ["/sbin/my_init"] -RUN apt-get update && apt-get -y upgrade && apt-get -y install \ - nano \ - rsync \ - software-properties-common \ - sudo \ - wget && \ - apt-get purge -y syslog-ng-core && \ - apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - ARG DOKKU_TAG=0.17.7 ARG DOKKU_GID=200 ARG DOKKU_UID=200 -ARG DOKKU_DOCKERFILE=true -ARG DOKKU_WEB_CONFIG=false ARG DOKKU_HOSTNAME=dokku.invalid -ARG DOKKU_VHOST_ENABLE=false ARG DOKKU_SKIP_KEY_FILE=true +ARG DOKKU_VHOST_ENABLE=false +ARG DOKKU_WEB_CONFIG=false RUN addgroup --gid $DOKKU_GID dokku && \ adduser --uid $DOKKU_UID --gid $DOKKU_GID --disabled-password --gecos "" "dokku" -RUN export DOKKU_TAG=$DOKKU_TAG \ - DOKKU_DOCKERFILE=$DOKKU_DOCKERFILE \ - DOKKU_WEB_CONFIG=$DOKKU_WEB_CONFIG \ - DOKKU_HOSTNAME=$DOKKU_HOSTNAME \ - DOKKU_VHOST_ENABLE=$DOKKU_VHOST_ENABLE \ - DOKKU_SKIP_KEY_FILE=$DOKKU_SKIP_KEY_FILE && \ - wget https://raw.githubusercontent.com/dokku/dokku/v$DOKKU_TAG/bootstrap.sh && \ - bash bootstrap.sh && \ - apt-get clean && rm -rf bootstrap.sh /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN echo "dokku dokku/hostname string $DOKKU_HOSTNAME" | debconf-set-selections && \ + echo "dokku dokku/skip_key_file boolean $DOKKU_SKIP_KEY_FILE" | debconf-set-selections && \ + echo "dokku dokku/vhost_enable boolean $DOKKU_VHOST_ENABLE" | debconf-set-selections && \ + echo "dokku dokku/web_config boolean $DOKKU_WEB_CONFIG" | debconf-set-selections && \ + curl -sSL https://packagecloud.io/dokku/dokku/gpgkey | apt-key add - && \ + echo "deb https://packagecloud.io/dokku/dokku/ubuntu/ bionic main" | tee /etc/apt/sources.list.d/dokku.list && \ + apt-get update && apt-get -y upgrade && apt-get --no-install-recommends -qq -y install \ + dokku \ + rsync && \ + apt-get purge -y syslog-ng-core && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* WORKDIR /tmp From 7aeb17986af39d40bfa024f458a38cc84f61f615 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 10 Jun 2019 17:02:49 -0400 Subject: [PATCH 15/27] docs: clarify usage of docker-based installation ssh-key addition wasn't previously scoped out. --- docs/getting-started/install/docker.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md index 524bb548a..fbab3528f 100644 --- a/docs/getting-started/install/docker.md +++ b/docs/getting-started/install/docker.md @@ -3,7 +3,7 @@ Pull the dokku/dokku image: ```shell -docker pull dokku/dokku:v0.17.6 +docker pull dokku/dokku:v0.17.7 ``` Next, run the image. @@ -17,16 +17,16 @@ docker run \ --publish 8443:443 \ --volume /mount:/mount \ --volume /var/run/docker.sock:/var/run/docker.sock \ - dokku/dokku:v0.17.6 + dokku/dokku:v0.17.7 ``` Dokku is run with the following options: - The global hostname is set to `dokku.me` on boot. - The container name is dokku. -- SSH port 22 is exposed on the host as 3022. -- HTTP port 80 is exposed on the host as 8000. -- HTTPS port 443 is exposed on the host as 8443. +- Container SSH port 22 is exposed on the host as 3022. +- Container HTTP port 80 is exposed on the host as 8000. +- Container HTTPS port 443 is exposed on the host as 8443. - Data within the container is stored on the host within the `/mount` directory. - The docker socket is mounted into container @@ -34,14 +34,21 @@ Application repositories, plugin config, and plugin data is persisted to disk wi To install custom plugins, create a `plugin-list` file in the host's `/mount` directory. The plugins listed herein will be automatically installed by Dokku on container boot. This file should be the following format: -``` +```yaml plugin_name: repository_url ``` An example for installing the postgres and redis plugins follows: -```shell +```yaml postgres: https://github.com/dokku/dokku-postgres.git redis: https://github.com/dokku/dokku-redis.git ``` +To initialize ssh-keys within the container, use `docker exec` to enter the container and run the appropriate ssh-keys commands. + +```shell +docker exec -it dokku bash +``` + +Please see the [user management documentation](/docs/deployment/user-management.md) for more information. From ecfb54c668f20ba50ee6ab7a87b0aaf505a02e30 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 10 Jun 2019 17:42:37 -0400 Subject: [PATCH 16/27] refactor: switch to /mnt/dokku --- Dockerfile | 8 ++++---- docker/etc/my_init.d/10_dokku_init | 20 ++++++++++---------- docs/getting-started/install/docker.md | 8 ++++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1709b14e3..0032a9c75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,10 +39,10 @@ RUN \ mv /home/dokku /skel/home/dokku && \ mv /var/lib/dokku/config /skel/var/lib/dokku/config && \ mv /var/lib/dokku/data /skel/var/lib/dokku/data && \ - ln -sf /mount/etc/ssh /etc/ssh && \ - ln -sf /mount/home/dokku /home/dokku && \ - ln -sf /mount/var/lib/dokku/config /var/lib/dokku/config && \ - ln -sf /mount/var/lib/dokku/data /var/lib/dokku/data && \ + ln -sf /mnt/dokku/etc/ssh /etc/ssh && \ + ln -sf /mnt/dokku/home/dokku /home/dokku && \ + ln -sf /mnt/dokku/var/lib/dokku/config /var/lib/dokku/config && \ + ln -sf /mnt/dokku/var/lib/dokku/data /var/lib/dokku/data && \ mv /etc/my_init.d/00_regen_ssh_host_keys.sh \ /etc/my_init.d/15_regen_ssh_host_keys && \ rm /etc/nginx/sites-enabled/default && \ diff --git a/docker/etc/my_init.d/10_dokku_init b/docker/etc/my_init.d/10_dokku_init index a8313c13a..21c562149 100755 --- a/docker/etc/my_init.d/10_dokku_init +++ b/docker/etc/my_init.d/10_dokku_init @@ -13,16 +13,16 @@ log-fail() { } main() { - if [[ ! -d /mount ]]; then - log-info "Creating missing /mount" - mkdir -p /mount + if [[ ! -d /mnt/dokku ]]; then + log-info "Creating missing /mnt/dokku" + mkdir -p /mnt/dokku fi directories=("/etc/ssh" "/home/dokku" "/var/lib/dokku/config" "/var/lib/dokku/data") for dir in "${directories[@]}"; do if [[ ! -e "$dir" ]]; then log-info "Copying ${dir} skel into place" - mkdir -p "/mount${dir}" + mkdir -p "/mnt/dokku${dir}" rsync -a "/skel${dir}/" "${dir}/" || log-fail "Unable to copy ${dir} skel into place" fi @@ -32,10 +32,10 @@ main() { fi done - if [[ -f /mount/plugin-list ]]; then + if [[ -f /mnt/dokku/plugin-list ]]; then while read line; do dokku plugin:install "$(echo "$line" | awk '{print $2}')" "$(echo "$line" | cut -d':' -f1)" - done Date: Mon, 10 Jun 2019 15:55:18 -0700 Subject: [PATCH 17/27] fix port typo --- docs/getting-started/install/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md index bf5f92b48..cb03f5622 100644 --- a/docs/getting-started/install/docker.md +++ b/docs/getting-started/install/docker.md @@ -25,7 +25,7 @@ Dokku is run with the following options: - The global hostname is set to `dokku.me` on boot. - The container name is dokku. - Container SSH port 22 is exposed on the host as 3022. -- Container HTTP port 80 is exposed on the host as 8000. +- Container HTTP port 80 is exposed on the host as 8080. - Container HTTPS port 443 is exposed on the host as 8443. - Data within the container is stored on the host within the `/var/lib/dokku` directory. - The docker socket is mounted into container From d8ada6a6e9440aadbaa97ce8e4978e535c71b9d9 Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Tue, 11 Jun 2019 11:32:41 -0700 Subject: [PATCH 18/27] install dokku from built deb package and use docker image for integrations tests --- .circleci/config.yml | 13 ++++---- .dockerignore | 1 + Dockerfile | 66 ++++++++++++++++++------------------- contrib/dokku-docker-bin.sh | 9 +++++ contrib/release-dokku | 11 +++++-- debian/postinst | 1 + rpm/dokku.postinst | 1 + tests.mk | 28 ++++------------ tests/ci/setup.sh | 61 +++++++++++++++++++--------------- 9 files changed, 102 insertions(+), 89 deletions(-) create mode 100755 contrib/dokku-docker-bin.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 41fe25837..130247282 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,9 +3,16 @@ jobs: build: machine: image: ubuntu-1604:201903-01 + environment: + DOKKU_HOST: dokku.me + DOKKU_PORT: 3022 parallelism: 4 steps: - checkout + - run: | + make ci-dependencies + - run: | + ./tests/ci/setup.sh - run: | if [[ "$(lsb_release -rs)" == "14.04" ]]; then # dokku.me now resolves to 10.0.0.2. add 10.0.0.2/24 to eth0 @@ -14,12 +21,6 @@ jobs: # dokku.me now resolves to 10.0.0.2. add 10.0.0.2/24 to ens4 sudo ip a a 10.0.0.2/24 broadcast 10.0.0.255 dev ens4 fi - - run: | - make ci-dependencies - - run: | - ./tests/ci/setup.sh - - run: | - echo 'export DOKKU_SKIP_CLEANUP=true' | sudo tee /home/dokku/.dokkurc/dokku_skip_cleanup - run: | case $CIRCLE_NODE_INDEX in 0) sudo -E make -e lint go-tests ci-go-coverage ;; diff --git a/.dockerignore b/.dockerignore index a89678a7f..29e2c3436 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ contrib build docs tests +!build/dokku.deb !contrib/dokku-installer.py !contrib/dokku-update !contrib/bash-completion diff --git a/Dockerfile b/Dockerfile index 0032a9c75..3dc52667a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,43 +10,43 @@ ARG DOKKU_SKIP_KEY_FILE=true ARG DOKKU_VHOST_ENABLE=false ARG DOKKU_WEB_CONFIG=false -RUN addgroup --gid $DOKKU_GID dokku && \ - adduser --uid $DOKKU_UID --gid $DOKKU_GID --disabled-password --gecos "" "dokku" +COPY ./build/dokku.deb /tmp/dokku.deb -RUN echo "dokku dokku/hostname string $DOKKU_HOSTNAME" | debconf-set-selections && \ - echo "dokku dokku/skip_key_file boolean $DOKKU_SKIP_KEY_FILE" | debconf-set-selections && \ - echo "dokku dokku/vhost_enable boolean $DOKKU_VHOST_ENABLE" | debconf-set-selections && \ - echo "dokku dokku/web_config boolean $DOKKU_WEB_CONFIG" | debconf-set-selections && \ - curl -sSL https://packagecloud.io/dokku/dokku/gpgkey | apt-key add - && \ - echo "deb https://packagecloud.io/dokku/dokku/ubuntu/ bionic main" | tee /etc/apt/sources.list.d/dokku.list && \ - apt-get update && apt-get -y upgrade && apt-get --no-install-recommends -qq -y install \ - dokku \ - rsync && \ - apt-get purge -y syslog-ng-core && \ - apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN addgroup --gid $DOKKU_GID dokku \ + && adduser --uid $DOKKU_UID --gid $DOKKU_GID --disabled-password --gecos "" "dokku" + +RUN echo "dokku dokku/hostname string $DOKKU_HOSTNAME" | debconf-set-selections \ + && echo "dokku dokku/skip_key_file boolean $DOKKU_SKIP_KEY_FILE" | debconf-set-selections \ + && echo "dokku dokku/vhost_enable boolean $DOKKU_VHOST_ENABLE" | debconf-set-selections \ + && echo "dokku dokku/web_config boolean $DOKKU_WEB_CONFIG" | debconf-set-selections \ + && curl -sSL https://packagecloud.io/dokku/dokku/gpgkey | apt-key add - \ + && echo "deb https://packagecloud.io/dokku/dokku/ubuntu/ bionic main" | tee /etc/apt/sources.list.d/dokku.list \ + && apt-get update -qq \ + && apt-get install --no-install-recommends --only-upgrade -y -qq openssl openssh-server \ + && apt --no-install-recommends -qq -y install rsync /tmp/dokku.deb \ + && apt-get purge -y -q syslog-ng-core \ + && apt-get autoremove -y -q \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* WORKDIR /tmp COPY ./docker . RUN \ - rsync -a /tmp/ / && \ - rm -rf /tmp/* && \ - rm /etc/runit/runsvdir/default/sshd/down && \ - chown -R dokku:dokku /home/dokku/ && \ - mkdir -p /skel/etc /skel/home /skel/var/lib/dokku /var/log/services && \ - mv /etc/ssh /skel/etc/ssh && \ - mv /home/dokku /skel/home/dokku && \ - mv /var/lib/dokku/config /skel/var/lib/dokku/config && \ - mv /var/lib/dokku/data /skel/var/lib/dokku/data && \ - ln -sf /mnt/dokku/etc/ssh /etc/ssh && \ - ln -sf /mnt/dokku/home/dokku /home/dokku && \ - ln -sf /mnt/dokku/var/lib/dokku/config /var/lib/dokku/config && \ - ln -sf /mnt/dokku/var/lib/dokku/data /var/lib/dokku/data && \ - mv /etc/my_init.d/00_regen_ssh_host_keys.sh \ - /etc/my_init.d/15_regen_ssh_host_keys && \ - rm /etc/nginx/sites-enabled/default && \ - rm /usr/share/nginx/html/index.html && \ - rm /etc/my_init.d/10_syslog-ng.init && \ - sed -i '/imklog/d' /etc/rsyslog.conf && \ - rm /var/log/btmp /var/log/wtmp /var/log/*log /var/log/apt/* /var/log/dokku/* /var/log/nginx/* + rsync -a /tmp/ / \ + && rm -rf /tmp/* \ + && rm /etc/runit/runsvdir/default/sshd/down \ + && chown -R dokku:dokku /home/dokku/ \ + && mkdir -p /skel/etc /skel/home /skel/var/lib/dokku /var/log/services \ + && mv /etc/ssh /skel/etc/ssh \ + && mv /home/dokku /skel/home/dokku \ + && mv /var/lib/dokku/config /skel/var/lib/dokku/config \ + && mv /var/lib/dokku/data /skel/var/lib/dokku/data \ + && ln -sf /mnt/dokku/etc/ssh /etc/ssh \ + && ln -sf /mnt/dokku/home/dokku /home/dokku \ + && ln -sf /mnt/dokku/var/lib/dokku/config /var/lib/dokku/config \ + && ln -sf /mnt/dokku/var/lib/dokku/data /var/lib/dokku/data \ + && mv /etc/my_init.d/00_regen_ssh_host_keys.sh /etc/my_init.d/15_regen_ssh_host_keys \ + && rm /etc/nginx/sites-enabled/default /usr/share/nginx/html/index.html /etc/my_init.d/10_syslog-ng.init \ + && sed -i '/imklog/d' /etc/rsyslog.conf \ + && rm /var/log/btmp /var/log/wtmp /var/log/*log /var/log/apt/* /var/log/dokku/* /var/log/nginx/* diff --git a/contrib/dokku-docker-bin.sh b/contrib/dokku-docker-bin.sh new file mode 100755 index 000000000..f20ae5928 --- /dev/null +++ b/contrib/dokku-docker-bin.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x && DOKKU_DOCKER_ENV="--env DOKKU_TRACE=on" +DOKKU_DOCKER_CONTAINER_NAME=${DOKKU_DOCKER_CONTAINER_NAME:=dokku} + +# TODO: handle cases where we need a tty +DOCKER_CMD="docker exec $DOKKU_DOCKER_ENV -i $DOKKU_DOCKER_CONTAINER_NAME dokku $*" + +exec "$DOCKER_CMD" diff --git a/contrib/release-dokku b/contrib/release-dokku index 3fa7b3890..f56d4e564 100755 --- a/contrib/release-dokku +++ b/contrib/release-dokku @@ -319,6 +319,13 @@ fn-require-bin() { fi } +fn-build-docker-image() { + declare desc="Builds the dokku docker image and tags it with the given version" + declare VERSION="$1" + docker build -t "dokku/dokku:$VERSION" . + docker push "dokku/dokku:$VERSION" +} + main() { declare RELEASE="$1" local CURRENT_VERSION NEXT_VERSION IS_RELEASE @@ -360,14 +367,14 @@ main() { if [[ "$RELEASE" != "build" ]]; then fn-publish-package "$IS_RELEASE" "deb" "build/dokku_${NEXT_VERSION}_amd64.deb" || log-fail "Error publishing deb package" fn-publish-package "$IS_RELEASE" "rpm" "build/dokku-${NEXT_VERSION}-1.x86_64.rpm" || log-fail "Error publishing rpm package" - docker build -t "dokku/dokku:$NEXT_VERSION" . - docker push "dokku/dokku:$NEXT_VERSION" + fn-build-docker-image "$NEXT_VERSION" || log-fail "Error building docker image" fn-repo-push-tags "$IS_RELEASE" fi if [[ "$IS_RELEASE" != "true" ]]; then git reset -q HEAD plugins/*/plugin.toml git checkout -- plugins/*/plugin.toml + cp "build/dokku_${NEXT_VERSION}_amd64.deb" build/dokku.deb fi echo "build/dokku_${NEXT_VERSION}_amd64.deb" >build/deb-filename diff --git a/debian/postinst b/debian/postinst index 014926a76..0cc555dda 100755 --- a/debian/postinst +++ b/debian/postinst @@ -40,6 +40,7 @@ case "$1" in egrep -i "^docker" /etc/group || groupadd docker usermod -aG docker dokku + mkdir -p ${DOKKU_ROOT}/.dokkurc echo "Setting up storage directories" mkdir -p ${DOKKU_LIB_ROOT}/data ${DOKKU_LIB_ROOT}/data/storage diff --git a/rpm/dokku.postinst b/rpm/dokku.postinst index 4658c75b1..182e645c0 100644 --- a/rpm/dokku.postinst +++ b/rpm/dokku.postinst @@ -9,6 +9,7 @@ setup-user() { touch "$DOKKU_ROOT/.ssh/authorized_keys" chown -R dokku:dokku "$DOKKU_ROOT/.ssh" usermod -aG docker dokku + mkdir -p "$DOKKU_ROOT/.dokkurc" } setup-storage() { diff --git a/tests.mk b/tests.mk index c42dc22b3..f93f97ae5 100644 --- a/tests.mk +++ b/tests.mk @@ -45,10 +45,9 @@ endif ci-dependencies: bats shellcheck shfmt xmlstarlet setup-deploy-tests: - mkdir -p /home/dokku ifdef ENABLE_DOKKU_TRACE echo "-----> Enabling tracing" - echo "export DOKKU_TRACE=1" >> /home/dokku/dokkurc + docker exec -ti dokku bash -c 'echo "export DOKKU_TRACE=1" >> /home/dokku/dokkurc' endif @echo "Setting dokku.me in /etc/hosts" sudo /bin/bash -c "[[ `ping -c1 dokku.me >/dev/null 2>&1; echo $$?` -eq 0 ]] || echo \"127.0.0.1 dokku.me *.dokku.me www.test.app.dokku.me\" >> /etc/hosts" @@ -61,33 +60,20 @@ endif @echo "-----> Setting up ssh config..." ifneq ($(shell ls /root/.ssh/config >/dev/null 2>&1 ; echo $$?),0) - echo "Host dokku.me \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config + echo "Host dokku.me \\r\\n Port 3022 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config echo "Host 127.0.0.1 \\r\\n Port 22333 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config else ifeq ($(shell grep dokku.me /root/.ssh/config),) - echo "Host dokku.me \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config + echo "Host dokku.me \\r\\n Port 3022 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config echo "Host 127.0.0.1 \\r\\n Port 22333 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config endif -ifneq ($(wildcard /etc/ssh/sshd_config),) - sed --in-place "s/^#Port 22$\/Port 22/g" /etc/ssh/sshd_config -ifeq ($(shell grep 22333 /etc/ssh/sshd_config),) - sed --in-place "s:^Port 22:Port 22 \\nPort 22333:g" /etc/ssh/sshd_config -endif - service ssh restart -endif - @echo "-----> Installing SSH public key..." - sudo sshcommand acl-remove dokku test - cat /root/.ssh/dokku_test_rsa.pub | sudo sshcommand acl-add dokku test + docker exec -ti dokku bash -c "sshcommand acl-remove dokku test" + docker exec -ti dokku bash -c "echo `cat /root/.ssh/dokku_test_rsa.pub` | sshcommand acl-add dokku test" @echo "-----> Intitial SSH connection to populate known_hosts..." - ssh -o StrictHostKeyChecking=no dokku@dokku.me help >/dev/null - ssh -o StrictHostKeyChecking=no dokku@127.0.0.1 help >/dev/null - -ifeq ($(shell grep dokku.me /home/dokku/VHOST 2>/dev/null),) - @echo "-----> Setting default VHOST to dokku.me..." - echo "dokku.me" > /home/dokku/VHOST -endif + sudo ssh -o StrictHostKeyChecking=no dokku@dokku.me help >/dev/null + sudo ssh -o StrictHostKeyChecking=no dokku@127.0.0.1 help >/dev/null lint-setup: @mkdir -p test-results/shellcheck tmp/shellcheck diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 190f2812e..0eeeee7be 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -4,6 +4,8 @@ set -eo pipefail readonly ROOT_DIR="$(cd "$(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")" && pwd)" install_dependencies() { + echo "=====> install_dependencies on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" + mkdir -p "$ROOT_DIR/build/" HEROKUISH_VERSION=$(grep HEROKUISH_VERSION "${ROOT_DIR}/deb.mk" | head -n1 | cut -d' ' -f3) HEROKUISH_PACKAGE_NAME="herokuish_${HEROKUISH_VERSION}_amd64.deb" @@ -25,54 +27,59 @@ install_dependencies() { PROCFILE_UTIL_PACKAGE_NAME="procfile-util_${PROCFILE_VERSION}_amd64.deb" curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/trusty/procfile-util_${PROCFILE_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${PROCFILE_UTIL_PACKAGE_NAME}" - sudo add-apt-repository -y ppa:nginx/stable - sudo apt-get update - sudo apt-get -qq -y install cgroupfs-mount nginx - sudo cp "${ROOT_DIR}/tests/dhparam.pem" /etc/nginx/dhparam.pem + sudo dpkg -i "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$SSHCOMMAND_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$SIGIL_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" +} - sudo dpkg -i "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" - sudo dpkg -i "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" - sudo dpkg -i "${ROOT_DIR}/build/$SSHCOMMAND_PACKAGE_NAME" - sudo dpkg -i "${ROOT_DIR}/build/$SIGIL_PACKAGE_NAME" - sudo dpkg -i "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" +build_dokku() { + echo "=====> build_dokku on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" + "${ROOT_DIR}/contrib/release-dokku" build + docker build -t dokku/dokku:test . +} + +run_dokku_container() { + echo "=====> run_dokku_container on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" + docker run -d \ + --env DOKKU_HOSTNAME=dokku.me \ + --name dokku \ + --publish 3022:22 \ + --publish 22333:22 \ + --publish 80:80 \ + --publish 443:443 \ + --volume /var/lib/dokku:/mnt/dokku \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + dokku/dokku:test + + sleep 1 && docker logs dokku } install_dokku() { + echo "=====> install_dokku on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" + if [[ "$FROM_SOURCE" == "true" ]]; then sudo -E CI=true make -e install return fi - "${ROOT_DIR}/contrib/release-dokku" build + build_dokku + run_dokku_container - echo "dokku dokku/hostname string dokku.me" | sudo debconf-set-selections - echo "dokku dokku/key_file string /root/.ssh/id_rsa.pub" | sudo debconf-set-selections - echo "dokku dokku/nginx_enable boolean true" | sudo debconf-set-selections - echo "dokku dokku/skip_key_file boolean true" | sudo debconf-set-selections - echo "dokku dokku/vhost_enable boolean true" | sudo debconf-set-selections - echo "dokku dokku/web_config boolean false" | sudo debconf-set-selections - sudo dpkg -i "$(cat "${ROOT_DIR}/build/deb-filename")" + sudo mkdir -p /usr/local/bin + sudo cp -v contrib/dokku-docker-bin.sh /usr/local/bin/dokku } # shellcheck disable=SC2120 setup_circle() { echo "=====> setup_circle on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" - sudo -E CI=true make -e sshcommand - # need to add the dokku user to the docker group - sudo usermod -G docker dokku - [[ "$1" == "buildstack" ]] && BUILD_STACK=true make -e stack install_dependencies install_dokku sudo -E make -e setup-deploy-tests - bash --version - docker version lsb_release -a - # setup .dokkurc - sudo -E mkdir -p /home/dokku/.dokkurc - sudo -E chown dokku:ubuntu /home/dokku/.dokkurc - sudo -E chmod 775 /home/dokku/.dokkurc # pull node:4 image for testing sudo docker pull node:4 } From f723333a905aa7a959bdb36bc94575dcca7ca850 Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Tue, 11 Jun 2019 11:36:17 -0700 Subject: [PATCH 19/27] don't run shfmt in CI --- .circleci/config.yml | 2 +- tests.mk | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 130247282..94ed85f1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: fi - run: | case $CIRCLE_NODE_INDEX in - 0) sudo -E make -e lint go-tests ci-go-coverage ;; + 0) sudo -E make -e lint-ci go-tests ci-go-coverage ;; 1) sudo -E make -e deploy-test-checks-root deploy-test-config ;; 2) sudo -E make -e deploy-test-multi ;; 3) sudo -E make -e deploy-test-go-fail-predeploy deploy-test-go-fail-postdeploy ;; diff --git a/tests.mk b/tests.mk index f93f97ae5..3465ff805 100644 --- a/tests.mk +++ b/tests.mk @@ -42,7 +42,7 @@ else endif endif -ci-dependencies: bats shellcheck shfmt xmlstarlet +ci-dependencies: bats shellcheck xmlstarlet setup-deploy-tests: ifdef ENABLE_DOKKU_TRACE @@ -80,16 +80,19 @@ lint-setup: @find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' > tmp/shellcheck/test-files @cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' | cut -d' ' -f2 | paste -d, -s > tmp/shellcheck/exclude -lint: lint-setup - # verifying via shfmt - # shfmt -l -bn -ci -i 2 -d . - @shfmt -l -bn -ci -i 2 -d . - +lint-ci: lint-setup # these are disabled due to their expansive existence in the codebase. we should clean it up though @cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' @echo linting... @cat tmp/shellcheck/test-files | xargs shellcheck -e $(shell cat tmp/shellcheck/exclude) | tests/shellcheck-to-junit --output test-results/shellcheck/results.xml --files tmp/shellcheck/test-files --exclude $(shell cat tmp/shellcheck/exclude) +lint-shfmt: shfmt + # verifying via shfmt + # shfmt -l -bn -ci -i 2 -d . + @shfmt -l -bn -ci -i 2 -d . + +lint: lint-shfmt lint-ci + ci-go-coverage: docker run --rm -ti \ -e DOKKU_ROOT=/home/dokku \ From 7de68e70529ed85cac369ae83c7c48d0ae85af56 Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Tue, 11 Jun 2019 11:54:06 -0700 Subject: [PATCH 20/27] remove unnecessary CI env vars --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 94ed85f1b..5945d27a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,9 +3,6 @@ jobs: build: machine: image: ubuntu-1604:201903-01 - environment: - DOKKU_HOST: dokku.me - DOKKU_PORT: 3022 parallelism: 4 steps: - checkout From e8382362e5017ea5a93b93137b76821deee66fda Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Tue, 11 Jun 2019 15:15:05 -0700 Subject: [PATCH 21/27] remove tests/dhparam.pem dockerignore exception --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 29e2c3436..977ed7878 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,5 +11,4 @@ tests !contrib/dokku-update !contrib/bash-completion !contrib/docker -!tests/dhparam.pem vendor From 67bdd470156b684b502c5f67e934baf22cb8d17c Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Tue, 11 Jun 2019 15:15:22 -0700 Subject: [PATCH 22/27] wait for CI dokku container to startup --- Dockerfile | 1 + contrib/dokku-docker-bin.sh | 5 ++--- tests/ci/setup.sh | 23 +++++++++++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3dc52667a..bc0ee4e8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM phusion/baseimage:0.11 CMD ["/sbin/my_init"] +ARG DOCKER_BIN=/usr/local/bin/docker ARG DOKKU_TAG=0.17.7 ARG DOKKU_GID=200 ARG DOKKU_UID=200 diff --git a/contrib/dokku-docker-bin.sh b/contrib/dokku-docker-bin.sh index f20ae5928..3bc362322 100755 --- a/contrib/dokku-docker-bin.sh +++ b/contrib/dokku-docker-bin.sh @@ -4,6 +4,5 @@ set -eo pipefail DOKKU_DOCKER_CONTAINER_NAME=${DOKKU_DOCKER_CONTAINER_NAME:=dokku} # TODO: handle cases where we need a tty -DOCKER_CMD="docker exec $DOKKU_DOCKER_ENV -i $DOKKU_DOCKER_CONTAINER_NAME dokku $*" - -exec "$DOCKER_CMD" +# shellcheck disable=SC2086 +docker exec $DOKKU_DOCKER_ENV -i "$DOKKU_DOCKER_CONTAINER_NAME" dokku "$@" diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 0eeeee7be..d2efe38b2 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -53,7 +53,26 @@ run_dokku_container() { --volume /var/run/docker.sock:/var/run/docker.sock \ dokku/dokku:test - sleep 1 && docker logs dokku + check_container +} + +check_container() { + echo "=====> check_container on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" + local is_up + local cnt=0 + while true; do + echo "$(date) [count: $cnt]: waiting for dokku startup" + is_up=$( + docker exec -ti dokku ps -ef | grep "/usr/sbin/sshd -D" >/dev/null 2>&1 + echo $? + ) + if [[ $is_up -eq 0 ]]; then + echo "" && docker logs dokku + break + fi + sleep 2 + cnt=$((cnt + 1)) + done } install_dokku() { @@ -68,7 +87,7 @@ install_dokku() { run_dokku_container sudo mkdir -p /usr/local/bin - sudo cp -v contrib/dokku-docker-bin.sh /usr/local/bin/dokku + sudo cp -fv contrib/dokku-docker-bin.sh /usr/local/bin/dokku } # shellcheck disable=SC2120 From 8830e15526685e2e76eec11169b41e839818c06b Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Tue, 11 Jun 2019 18:03:21 -0700 Subject: [PATCH 23/27] use custom docker bin --- Dockerfile | 1 - docker/etc/my_init.d/10_dokku_init | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bc0ee4e8e..3dc52667a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,6 @@ FROM phusion/baseimage:0.11 CMD ["/sbin/my_init"] -ARG DOCKER_BIN=/usr/local/bin/docker ARG DOKKU_TAG=0.17.7 ARG DOKKU_GID=200 ARG DOKKU_UID=200 diff --git a/docker/etc/my_init.d/10_dokku_init b/docker/etc/my_init.d/10_dokku_init index 21c562149..7caff134b 100755 --- a/docker/etc/my_init.d/10_dokku_init +++ b/docker/etc/my_init.d/10_dokku_init @@ -44,6 +44,11 @@ main() { dokku --quiet domains:set-global "$DOKKU_HOSTNAME" fi + if [[ -x /usr/local/bin/docker ]]; then + mkdir -p /home/dokku/.dokkurc + echo "export DOCKER_BIN=/usr/local/bin/docker" >"/home/dokku/.dokkurc/DOCKER_BIN" + fi + if [[ ! -f /mnt/dokku/etc/nginx/dhparam.pem ]]; then mkdir -p /mnt/dokku/etc/nginx openssl dhparam -out /mnt/dokku/etc/nginx/dhparam.pem 2048 From ded056d0036119376282a244ac63334c83efe518 Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Thu, 11 Jul 2019 14:30:49 -0700 Subject: [PATCH 24/27] hack: use tmp file for CHECKS file without EOF newline --- plugins/scheduler-docker-local/check-deploy | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/plugins/scheduler-docker-local/check-deploy b/plugins/scheduler-docker-local/check-deploy index 148c427f7..f3d8a96f8 100755 --- a/plugins/scheduler-docker-local/check-deploy +++ b/plugins/scheduler-docker-local/check-deploy @@ -87,7 +87,7 @@ scheduler-docker-local-check-deploy() { local CHECK_DEPLOY_TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${FUNCNAME[0]}.XXXX") "$DOCKER_BIN" cp "$DOKKU_APP_CONTAINER_ID:/app/CHECKS" "$CHECK_DEPLOY_TMP_WORK_DIR" 2>/dev/null || true - local FILENAME=${CHECK_DEPLOY_TMP_WORK_DIR}/CHECKS + local CHECKS_FILENAME=${CHECK_DEPLOY_TMP_WORK_DIR}/CHECKS checks_check_deploy_cleanup() { declare desc="cleans up CHECK_DEPLOY_TMP_WORK_DIR and print container output" @@ -101,7 +101,7 @@ scheduler-docker-local-check-deploy() { } trap "checks_check_deploy_cleanup $DOKKU_APP_CONTAINER_ID" RETURN INT TERM EXIT - if [[ ! -s "${CHECK_DEPLOY_TMP_WORK_DIR}/CHECKS" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]]; then + if [[ ! -s "${CHECKS_FILENAME}" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]]; then # We allow custom check for web instances only if [[ "$DOKKU_APP_CONTAINER_TYPE" == "web" ]]; then dokku_log_verbose "For more efficient zero downtime deployments, create a file CHECKS." @@ -131,12 +131,18 @@ scheduler-docker-local-check-deploy() { fi # ensure CHECKS file has trailing newline - if [[ "$(tail -c1 "$FILENAME")" != "" ]]; then - echo "" >>"$FILENAME" + if [[ "$(tail -c1 "$CHECKS_FILENAME")" != "" ]]; then + # workaround for CHECKS file when owner is root. seems to only happen when running inside docker + if [[ "$(stat -c %u "$CHECKS_FILENAME")" == "0" ]]; then + local NEW_CHECKS_FILENAME=${CHECK_DEPLOY_TMP_WORK_DIR}/$(mktemp dokku-${FUNCNAME[0]}-CHECKS.XXXX) + cat ${CHECKS_FILENAME} >${NEW_CHECKS_FILENAME} + CHECKS_FILENAME=$NEW_CHECKS_FILENAME + fi + echo "" >>"$CHECKS_FILENAME" fi # Reads name/value pairs, sets the WAIT and TIMEOUT variables - exec <"$FILENAME" + exec <"$CHECKS_FILENAME" local line local NAME local VALUE @@ -173,7 +179,7 @@ scheduler-docker-local-check-deploy() { local CURL_OPTIONS+=" -H X-Forwarded-Proto:https" fi - exec <"$FILENAME" + exec <"$CHECKS_FILENAME" local CHECK_URL local EXPECTED while read -r CHECK_URL EXPECTED; do From 1437b77c6ca2b3c7ab88c1f2984922f70bd92b4c Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Fri, 12 Jul 2019 10:05:31 -0700 Subject: [PATCH 25/27] run tests against native dokku --- .circleci/config.yml | 2 ++ tests.mk | 30 ++++++++++++++++++++------ tests/ci/setup.sh | 51 +++++++++++++++++++++++++++++++------------- 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5945d27a2..7bf80df43 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,8 @@ jobs: make ci-dependencies - run: | ./tests/ci/setup.sh + - run: | + echo 'export DOKKU_SKIP_CLEANUP=true' | sudo tee /home/dokku/.dokkurc/dokku_skip_cleanup - run: | if [[ "$(lsb_release -rs)" == "14.04" ]]; then # dokku.me now resolves to 10.0.0.2. add 10.0.0.2/24 to eth0 diff --git a/tests.mk b/tests.mk index 3465ff805..a42bee00c 100644 --- a/tests.mk +++ b/tests.mk @@ -1,4 +1,5 @@ SYSTEM := $(shell sh -c 'uname -s 2>/dev/null') +DOKKU_SSH_PORT ?= 22 bats: ifeq ($(SYSTEM),Darwin) @@ -47,7 +48,7 @@ ci-dependencies: bats shellcheck xmlstarlet setup-deploy-tests: ifdef ENABLE_DOKKU_TRACE echo "-----> Enabling tracing" - docker exec -ti dokku bash -c 'echo "export DOKKU_TRACE=1" >> /home/dokku/dokkurc' + dokku trace:on endif @echo "Setting dokku.me in /etc/hosts" sudo /bin/bash -c "[[ `ping -c1 dokku.me >/dev/null 2>&1; echo $$?` -eq 0 ]] || echo \"127.0.0.1 dokku.me *.dokku.me www.test.app.dokku.me\" >> /etc/hosts" @@ -60,20 +61,35 @@ endif @echo "-----> Setting up ssh config..." ifneq ($(shell ls /root/.ssh/config >/dev/null 2>&1 ; echo $$?),0) - echo "Host dokku.me \\r\\n Port 3022 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config + echo "Host dokku.me \\r\\n Port $(DOKKU_SSH_PORT) \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config echo "Host 127.0.0.1 \\r\\n Port 22333 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config else ifeq ($(shell grep dokku.me /root/.ssh/config),) - echo "Host dokku.me \\r\\n Port 3022 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config + echo "Host dokku.me \\r\\n Port $(DOKKU_SSH_PORT) \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config echo "Host 127.0.0.1 \\r\\n Port 22333 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config endif +ifneq ($(wildcard /etc/ssh/sshd_config),) + sed --in-place "s/^#Port 22$\/Port 22/g" /etc/ssh/sshd_config +ifeq ($(shell grep 22333 /etc/ssh/sshd_config),) + sed --in-place "s:^Port 22:Port 22 \\nPort 22333:g" /etc/ssh/sshd_config +endif + service ssh restart +endif + @echo "-----> Installing SSH public key..." - docker exec -ti dokku bash -c "sshcommand acl-remove dokku test" - docker exec -ti dokku bash -c "echo `cat /root/.ssh/dokku_test_rsa.pub` | sshcommand acl-add dokku test" + sudo sshcommand acl-remove dokku test + cat /root/.ssh/dokku_test_rsa.pub | sudo sshcommand acl-add dokku test + # docker exec -ti dokku bash -c "sshcommand acl-remove dokku test" + # docker exec -ti dokku bash -c "echo `cat /root/.ssh/dokku_test_rsa.pub` | sshcommand acl-add dokku test" @echo "-----> Intitial SSH connection to populate known_hosts..." - sudo ssh -o StrictHostKeyChecking=no dokku@dokku.me help >/dev/null - sudo ssh -o StrictHostKeyChecking=no dokku@127.0.0.1 help >/dev/null + ssh -o StrictHostKeyChecking=no dokku@dokku.me help >/dev/null + ssh -o StrictHostKeyChecking=no dokku@127.0.0.1 help >/dev/null + +ifeq ($(shell grep dokku.me /home/dokku/VHOST 2>/dev/null),) + @echo "-----> Setting default VHOST to dokku.me..." + echo "dokku.me" > /home/dokku/VHOST +endif lint-setup: @mkdir -p test-results/shellcheck tmp/shellcheck diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index d2efe38b2..c33bb30da 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -27,6 +27,11 @@ install_dependencies() { PROCFILE_UTIL_PACKAGE_NAME="procfile-util_${PROCFILE_VERSION}_amd64.deb" curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/trusty/procfile-util_${PROCFILE_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${PROCFILE_UTIL_PACKAGE_NAME}" + sudo add-apt-repository -y ppa:nginx/stable + sudo apt-get update + sudo apt-get -qq -y install nginx + sudo cp "${ROOT_DIR}/tests/dhparam.pem" /etc/nginx/dhparam.pem + sudo dpkg -i "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \ "${ROOT_DIR}/build/$SSHCOMMAND_PACKAGE_NAME" \ @@ -37,6 +42,29 @@ install_dependencies() { build_dokku() { echo "=====> build_dokku on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" "${ROOT_DIR}/contrib/release-dokku" build +} + +install_dokku() { + echo "=====> install_dokku on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" + + if [[ "$FROM_SOURCE" == "true" ]]; then + sudo -E CI=true make -e install + return + fi + + build_dokku + + echo "dokku dokku/hostname string dokku.me" | sudo debconf-set-selections + echo "dokku dokku/key_file string /root/.ssh/id_rsa.pub" | sudo debconf-set-selections + echo "dokku dokku/nginx_enable boolean true" | sudo debconf-set-selections + echo "dokku dokku/skip_key_file boolean true" | sudo debconf-set-selections + echo "dokku dokku/vhost_enable boolean true" | sudo debconf-set-selections + echo "dokku dokku/web_config boolean false" | sudo debconf-set-selections + sudo dpkg -i "$(cat "${ROOT_DIR}/build/deb-filename")" +} + +build_dokku_docker_image() { + echo "=====> build_dokku_docker_image on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" docker build -t dokku/dokku:test . } @@ -75,30 +103,23 @@ check_container() { done } -install_dokku() { - echo "=====> install_dokku on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" - - if [[ "$FROM_SOURCE" == "true" ]]; then - sudo -E CI=true make -e install - return - fi - - build_dokku - run_dokku_container - - sudo mkdir -p /usr/local/bin - sudo cp -fv contrib/dokku-docker-bin.sh /usr/local/bin/dokku -} - # shellcheck disable=SC2120 setup_circle() { echo "=====> setup_circle on CIRCLE_NODE_INDEX: $CIRCLE_NODE_INDEX" + sudo -E CI=true make -e sshcommand + # need to add the dokku user to the docker group + sudo usermod -G docker dokku + [[ "$1" == "buildstack" ]] && BUILD_STACK=true make -e stack install_dependencies install_dokku sudo -E make -e setup-deploy-tests lsb_release -a + # setup .dokkurc + sudo -E mkdir -p /home/dokku/.dokkurc + sudo -E chown dokku:ubuntu /home/dokku/.dokkurc + sudo -E chmod 775 /home/dokku/.dokkurc # pull node:4 image for testing sudo docker pull node:4 } From b8c22597750ba5414075d07af26111f721b9a99f Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Fri, 12 Jul 2019 11:27:20 -0700 Subject: [PATCH 26/27] test docker image --- .circleci/config.yml | 5 +++++ tests.mk | 30 +++++++++++++++++++++++------- tests/ci/setup.sh | 17 +++++++++++++---- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bf80df43..7638102fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,6 +31,11 @@ jobs: shell: /bin/bash command: sudo -E make -e test-ci no_output_timeout: 60m + - run: | + if [ "${CIRCLE_NODE_INDEX}" == "0" ]; then + ./tests/ci/setup.sh docker + DOKKU_SSH_PORT=3022 sudo -E make -e test-ci-docker + fi - store_artifacts: path: ./coverage.out - store_artifacts: diff --git a/tests.mk b/tests.mk index a42bee00c..c3dfc5fbd 100644 --- a/tests.mk +++ b/tests.mk @@ -47,7 +47,7 @@ ci-dependencies: bats shellcheck xmlstarlet setup-deploy-tests: ifdef ENABLE_DOKKU_TRACE - echo "-----> Enabling tracing" + echo "-----> Enable dokku trace" dokku trace:on endif @echo "Setting dokku.me in /etc/hosts" @@ -66,6 +66,9 @@ ifneq ($(shell ls /root/.ssh/config >/dev/null 2>&1 ; echo $$?),0) else ifeq ($(shell grep dokku.me /root/.ssh/config),) echo "Host dokku.me \\r\\n Port $(DOKKU_SSH_PORT) \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config echo "Host 127.0.0.1 \\r\\n Port 22333 \\r\\n RequestTTY yes \\r\\n IdentityFile /root/.ssh/dokku_test_rsa" >> /root/.ssh/config +else + sed --in-place 's/Port 22 \r/Port $(DOKKU_SSH_PORT) \r/g' /root/.ssh/config + cat /root/.ssh/config endif ifneq ($(wildcard /etc/ssh/sshd_config),) @@ -79,17 +82,28 @@ endif @echo "-----> Installing SSH public key..." sudo sshcommand acl-remove dokku test cat /root/.ssh/dokku_test_rsa.pub | sudo sshcommand acl-add dokku test - # docker exec -ti dokku bash -c "sshcommand acl-remove dokku test" - # docker exec -ti dokku bash -c "echo `cat /root/.ssh/dokku_test_rsa.pub` | sshcommand acl-add dokku test" - - @echo "-----> Intitial SSH connection to populate known_hosts..." - ssh -o StrictHostKeyChecking=no dokku@dokku.me help >/dev/null - ssh -o StrictHostKeyChecking=no dokku@127.0.0.1 help >/dev/null ifeq ($(shell grep dokku.me /home/dokku/VHOST 2>/dev/null),) @echo "-----> Setting default VHOST to dokku.me..." echo "dokku.me" > /home/dokku/VHOST endif +ifeq ($(DOKKU_SSH_PORT), 22) + $(MAKE) prime-ssh-known-hosts +endif + +setup-docker-deploy-tests: setup-deploy-tests +ifdef ENABLE_DOKKU_TRACE + echo "-----> Enable dokku trace" + docker exec -ti dokku bash -c "dokku trace:on" +endif + docker exec -ti dokku bash -c "sshcommand acl-remove dokku test" + docker exec -ti dokku bash -c "echo `cat /root/.ssh/dokku_test_rsa.pub` | sshcommand acl-add dokku test" + $(MAKE) prime-ssh-known-hosts + +prime-ssh-known-hosts: + @echo "-----> Intitial SSH connection to populate known_hosts..." + ssh -o StrictHostKeyChecking=no dokku@dokku.me help >/dev/null + ssh -o StrictHostKeyChecking=no dokku@127.0.0.1 help >/dev/null lint-setup: @mkdir -p test-results/shellcheck tmp/shellcheck @@ -247,3 +261,5 @@ test-ci: @mkdir -p test-results/bats @cd tests/unit && echo "executing tests: $(shell cd tests/unit ; circleci tests glob *.bats | circleci tests split --split-by=timings --timings-type=classname | xargs)" cd tests/unit && bats --formatter bats-format-junit -e -T -o ../../test-results/bats $(shell cd tests/unit ; circleci tests glob *.bats | circleci tests split --split-by=timings --timings-type=classname | xargs) + +test-ci-docker: setup-docker-deploy-tests deploy-test-checks-root deploy-test-config deploy-test-multi deploy-test-go-fail-predeploy deploy-test-go-fail-postdeploy diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index c33bb30da..76d0ae551 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -74,7 +74,6 @@ run_dokku_container() { --env DOKKU_HOSTNAME=dokku.me \ --name dokku \ --publish 3022:22 \ - --publish 22333:22 \ --publish 80:80 \ --publish 443:443 \ --volume /var/lib/dokku:/mnt/dokku \ @@ -124,6 +123,16 @@ setup_circle() { sudo docker pull node:4 } -# shellcheck disable=SC2119 -setup_circle -exit $? +case "$1" in + docker) + sudo /etc/init.d/nginx stop + build_dokku_docker_image + run_dokku_container + ;; + *) + # shellcheck disable=SC2119 + setup_circle + exit $? + ;; +esac + From 11241e500397f5679237a419e85ed9542a0e659d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 18 Jul 2019 01:15:44 -0400 Subject: [PATCH 27/27] fix: add missing cgroupfs-mount --- tests/ci/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index 76d0ae551..b5e8e5099 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -29,7 +29,7 @@ install_dependencies() { sudo add-apt-repository -y ppa:nginx/stable sudo apt-get update - sudo apt-get -qq -y install nginx + sudo apt-get -qq -y install cgroupfs-mount nginx sudo cp "${ROOT_DIR}/tests/dhparam.pem" /etc/nginx/dhparam.pem sudo dpkg -i "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \