From cc0843391f11cc8cc83c089aa10a911be2c13f7b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 10 May 2026 21:17:09 -0400 Subject: [PATCH 1/4] feat: add docker healthcheck to dokku container The official dokku/dokku image gains a HEALTHCHECK directive backed by a loopback-only HTTP endpoint at `127.0.0.1:18080/_dokku/health`. The endpoint reports 200 once first-boot bootstrap finishes, sshd and nginx are accepting connections, and `dokku ps:restore` completes; otherwise it returns 503. Changes are scoped to the Docker overlay and Dockerfile so debian-package installs are unaffected. --- Dockerfile | 3 ++ docker/etc/my_init.d/10_dokku_init | 3 ++ docker/etc/nginx/conf.d/zz-dokku-health.conf | 16 +++++++++ .../runsvdir/default/dokku-restore/finish | 8 +++-- docs/getting-started/install/docker.md | 33 ++++++++++++++++++- 5 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 docker/etc/nginx/conf.d/zz-dokku-health.conf diff --git a/Dockerfile b/Dockerfile index 616983a1d..6dca6ff8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,3 +81,6 @@ RUN \ && rm -f /usr/local/openresty/nginx/conf/sites-enabled/default /usr/share/openresty/html/index.html \ && sed -i '/imklog/d' /etc/rsyslog.conf \ && rm -f /var/log/btmp /var/log/wtmp /var/log/*log /var/log/apt/* /var/log/dokku/*.log /var/log/nginx/* /var/log/openresty/* + +HEALTHCHECK --interval=30s --timeout=10s --start-period=5m --retries=3 \ + CMD curl -fsS http://127.0.0.1:18080/_dokku/health || exit 1 diff --git a/docker/etc/my_init.d/10_dokku_init b/docker/etc/my_init.d/10_dokku_init index 38948d4af..c740a3a09 100755 --- a/docker/etc/my_init.d/10_dokku_init +++ b/docker/etc/my_init.d/10_dokku_init @@ -35,6 +35,9 @@ main() { support-userns + rm -f /var/run/dokku/ready + mkdir -p /var/run/dokku + if [[ ! -d /mnt/dokku ]]; then log-info "Creating missing /mnt/dokku" mkdir -p /mnt/dokku diff --git a/docker/etc/nginx/conf.d/zz-dokku-health.conf b/docker/etc/nginx/conf.d/zz-dokku-health.conf new file mode 100644 index 000000000..92c1475e3 --- /dev/null +++ b/docker/etc/nginx/conf.d/zz-dokku-health.conf @@ -0,0 +1,16 @@ +# Internal health endpoint for the official dokku/dokku Docker image. +# Bound to loopback so it is reachable only from inside the container +# and never conflicts with user app vhosts on 80/443. +server { + listen 127.0.0.1:18080; + server_name _; + access_log off; + default_type text/plain; + + location = /_dokku/health { + if (-f /var/run/dokku/ready) { + return 200 "ok\n"; + } + return 503 "not ready\n"; + } +} diff --git a/docker/etc/runit/runsvdir/default/dokku-restore/finish b/docker/etc/runit/runsvdir/default/dokku-restore/finish index ea80c20c0..84270a92c 100755 --- a/docker/etc/runit/runsvdir/default/dokku-restore/finish +++ b/docker/etc/runit/runsvdir/default/dokku-restore/finish @@ -10,6 +10,10 @@ if [[ "$1" -ne 0 ]]; then fi echo "Done restoring dokku apps." -exec sleep infinity -exit 0 +while ! (echo > /dev/tcp/127.0.0.1/22) >/dev/null 2>&1; do sleep 1; done +while ! (echo > /dev/tcp/127.0.0.1/80) >/dev/null 2>&1; do sleep 1; done +touch /var/run/dokku/ready +echo "Dokku is ready." + +exec sleep infinity diff --git a/docs/getting-started/install/docker.md b/docs/getting-started/install/docker.md index 489762f1b..b2d7b0a9d 100644 --- a/docs/getting-started/install/docker.md +++ b/docs/getting-started/install/docker.md @@ -44,7 +44,38 @@ services: restart: unless-stopped ``` -The above command will start a new docker container that is ready when a message similar to `Runit started as PID 12345` appears. +## Container readiness + +The image ships with a Docker `HEALTHCHECK` that flips from `starting` to `healthy` once first-boot bootstrap is complete (skel restored, `plugin-list` plugins installed, core install triggers fired), nginx and sshd are accepting connections, and `dokku ps:restore` has finished. Until those conditions hold, the container is considered unhealthy and dependent services should not yet send traffic. + +To check the current state from the host: + +```shell +docker inspect --format='{{.State.Health.Status}}' dokku +``` + +Internally, the check is backed by a loopback-only HTTP endpoint: + +```shell +docker exec dokku curl -fsS http://127.0.0.1:18080/_dokku/health +``` + +The endpoint binds to `127.0.0.1:18080` inside the container by design - it is not published to the host and does not interfere with user app vhosts on ports 80/443. + +Compose dependents can gate on the healthcheck via `depends_on` with `condition: service_healthy`: + +```yaml +services: + dokku: + image: dokku/dokku:0.38.2 + # ... + bootstrap: + image: alpine:3.20 + command: ["echo", "dokku is ready"] + depends_on: + dokku: + condition: service_healthy +``` Dokku is run in the following configuration: From 2a4393a63a1036c4e8de6475b305ead1acfdeb63 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 10 May 2026 21:39:43 -0400 Subject: [PATCH 2/4] test: cover dokku container healthcheck wiring Adds a bats lint test that guards the static wiring (nginx conf, dokku-restore finish-script ordering, my_init sentinel reset, and the Dockerfile HEALTHCHECK line) plus a docker smoke test that boots the built image, waits for the health flip, exercises the loopback endpoint, asserts the port is not published to the host, and verifies the negative path. The smoke test is invoked via a new `make test-image-healthcheck` target and runs automatically in the build-image action after `docker buildx --load`. --- .github/actions/build-image/action.yml | 6 ++ Makefile | 5 + tests/image/healthcheck.sh | 128 +++++++++++++++++++++++ tests/unit/docker-image-healthcheck.bats | 57 ++++++++++ 4 files changed, 196 insertions(+) create mode 100755 tests/image/healthcheck.sh create mode 100644 tests/unit/docker-image-healthcheck.bats diff --git a/.github/actions/build-image/action.yml b/.github/actions/build-image/action.yml index b64856f56..e25c4ec59 100644 --- a/.github/actions/build-image/action.yml +++ b/.github/actions/build-image/action.yml @@ -26,6 +26,12 @@ runs: shell: bash run: | docker buildx build \ + --load \ --progress plain \ --tag "dokku/dokku:latest" \ . + + - name: smoke test image + shell: bash + run: | + make test-image-healthcheck IMAGE_TAG=dokku/dokku:latest diff --git a/Makefile b/Makefile index 10408d299..6fafe4b3c 100644 --- a/Makefile +++ b/Makefile @@ -226,3 +226,8 @@ vagrant-acl-add: vagrant-dokku: vagrant ssh -- "sudo -H -u root bash -c 'dokku $(RUN_ARGS)'" + +IMAGE_TAG ?= dokku/dokku:latest + +test-image-healthcheck: + IMAGE_TAG=$(IMAGE_TAG) ./tests/image/healthcheck.sh diff --git a/tests/image/healthcheck.sh b/tests/image/healthcheck.sh new file mode 100755 index 000000000..49f114e1a --- /dev/null +++ b/tests/image/healthcheck.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE_TAG="${IMAGE_TAG:-dokku/dokku:latest}" +CONTAINER_NAME="${CONTAINER_NAME:-dokku-healthcheck-smoke}" +START_TIMEOUT_SECONDS="${START_TIMEOUT_SECONDS:-360}" +UNHEALTHY_TIMEOUT_SECONDS="${UNHEALTHY_TIMEOUT_SECONDS:-180}" + +log-info() { + echo "-----> $*" +} + +log-fail() { + echo " ! $*" >&2 + exit 1 +} + +cleanup() { + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +require-image() { + if ! docker image inspect "$IMAGE_TAG" >/dev/null 2>&1; then + log-fail "Image '$IMAGE_TAG' not found in local docker daemon. Build it first (eg. 'docker build -t $IMAGE_TAG .')." + fi +} + +start-container() { + log-info "Starting container '$CONTAINER_NAME' from image '$IMAGE_TAG'" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + docker run -d \ + --name "$CONTAINER_NAME" \ + --env DOKKU_HOSTNAME=dokku.me \ + -v /var/run/docker.sock:/var/run/docker.sock \ + "$IMAGE_TAG" >/dev/null +} + +wait-for-catchall-vhost() { + local deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + if docker exec "$CONTAINER_NAME" test -f /etc/nginx/conf.d/00-default-vhost.conf; then + return 0 + fi + sleep 1 + done + + log-fail "Catch-all default vhost /etc/nginx/conf.d/00-default-vhost.conf was never created. \ +Likely cause: a stale build/package/dokku-*.deb that predates the catch-all template shipped in \ +plugins/nginx-vhosts/templates/default-site.conf. Rebuild the deb before running this test." +} + +wait-for-healthy() { + local deadline=$((SECONDS + START_TIMEOUT_SECONDS)) + local status + while (( SECONDS < deadline )); do + status=$(docker inspect --format='{{.State.Health.Status}}' "$CONTAINER_NAME" 2>/dev/null || echo "missing") + if [[ "$status" == "healthy" ]]; then + log-info "Container reached healthy status" + return 0 + fi + sleep 5 + done + + log-info "Container failed to become healthy within ${START_TIMEOUT_SECONDS}s. Last logs:" + docker logs --tail 100 "$CONTAINER_NAME" || true + log-fail "Container '$CONTAINER_NAME' did not reach healthy status (last status: ${status:-unknown})" +} + +assert-endpoint-ok() { + log-info "Verifying /_dokku/health returns ok from inside the container" + local body + body=$(docker exec "$CONTAINER_NAME" curl -fsS http://127.0.0.1:18080/_dokku/health) + if [[ "$body" != "ok" ]]; then + log-fail "Expected health endpoint body 'ok', got: ${body}" + fi +} + +assert-port-not-published() { + log-info "Verifying port 18080 is not published to the host" + local ports + ports=$(docker inspect --format='{{json .NetworkSettings.Ports}}' "$CONTAINER_NAME") + if echo "$ports" | grep -q "18080/tcp"; then + log-fail "Port 18080/tcp should not be published to the host. Inspect output: $ports" + fi + + if curl --max-time 2 -fsS http://127.0.0.1:18080/_dokku/health >/dev/null 2>&1; then + log-fail "Health endpoint is reachable from the host on 127.0.0.1:18080 - it must be loopback-only inside the container." + fi +} + +assert-unhealthy-after-sentinel-removed() { + log-info "Removing /var/run/dokku/ready and confirming endpoint returns failure" + docker exec "$CONTAINER_NAME" rm -f /var/run/dokku/ready + + local exit_code=0 + docker exec "$CONTAINER_NAME" curl -fsS http://127.0.0.1:18080/_dokku/health >/dev/null 2>&1 || exit_code=$? + if (( exit_code == 0 )); then + log-fail "Health endpoint still returned success after the readiness sentinel was removed" + fi + + log-info "Waiting for docker to flip the health status to unhealthy" + local deadline=$((SECONDS + UNHEALTHY_TIMEOUT_SECONDS)) + local status + while (( SECONDS < deadline )); do + status=$(docker inspect --format='{{.State.Health.Status}}' "$CONTAINER_NAME" 2>/dev/null || echo "missing") + if [[ "$status" == "unhealthy" ]]; then + log-info "Container flipped to unhealthy" + return 0 + fi + sleep 5 + done + + log-fail "Container did not flip to unhealthy within ${UNHEALTHY_TIMEOUT_SECONDS}s (last status: ${status:-unknown})" +} + +main() { + require-image + start-container + wait-for-catchall-vhost + wait-for-healthy + assert-endpoint-ok + assert-port-not-published + assert-unhealthy-after-sentinel-removed + log-info "All healthcheck smoke tests passed." +} + +main "$@" diff --git a/tests/unit/docker-image-healthcheck.bats b/tests/unit/docker-image-healthcheck.bats new file mode 100644 index 000000000..e91a65829 --- /dev/null +++ b/tests/unit/docker-image-healthcheck.bats @@ -0,0 +1,57 @@ +#!/usr/bin/env bats + +REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + +NGINX_CONF="${REPO_ROOT}/docker/etc/nginx/conf.d/zz-dokku-health.conf" +FINISH_SCRIPT="${REPO_ROOT}/docker/etc/runit/runsvdir/default/dokku-restore/finish" +MY_INIT_SCRIPT="${REPO_ROOT}/docker/etc/my_init.d/10_dokku_init" +DOCKERFILE="${REPO_ROOT}/Dockerfile" + +@test "(docker-image) [healthcheck] nginx health conf is shipped via the docker overlay" { + [[ -f "$NGINX_CONF" ]] + grep -qE "listen[[:space:]]+127\.0\.0\.1:18080" "$NGINX_CONF" + grep -qE "location = /_dokku/health" "$NGINX_CONF" + grep -qE "/var/run/dokku/ready" "$NGINX_CONF" + grep -qE "default_type[[:space:]]+text/plain" "$NGINX_CONF" +} + +@test "(docker-image) [healthcheck] finish script touches sentinel after waiting for sshd and nginx" { + [[ -f "$FINISH_SCRIPT" ]] + + local sshd_line nginx_line touch_line + sshd_line=$(grep -nE "/dev/tcp/127\.0\.0\.1/22" "$FINISH_SCRIPT" | head -n1 | cut -d: -f1) + nginx_line=$(grep -nE "/dev/tcp/127\.0\.0\.1/80" "$FINISH_SCRIPT" | head -n1 | cut -d: -f1) + touch_line=$(grep -nE "touch /var/run/dokku/ready" "$FINISH_SCRIPT" | head -n1 | cut -d: -f1) + + [[ -n "$sshd_line" ]] + [[ -n "$nginx_line" ]] + [[ -n "$touch_line" ]] + (( touch_line > sshd_line )) + (( touch_line > nginx_line )) +} + +@test "(docker-image) [healthcheck] my_init clears stale sentinel on container boot" { + [[ -f "$MY_INIT_SCRIPT" ]] + + local rm_line mkdir_line plugin_line hostname_line + rm_line=$(grep -nE "rm -f /var/run/dokku/ready" "$MY_INIT_SCRIPT" | head -n1 | cut -d: -f1) + mkdir_line=$(grep -nE "mkdir -p /var/run/dokku( |$)" "$MY_INIT_SCRIPT" | head -n1 | cut -d: -f1) + plugin_line=$(grep -nE "plugin:install" "$MY_INIT_SCRIPT" | head -n1 | cut -d: -f1) + hostname_line=$(grep -nE "domains:set-global" "$MY_INIT_SCRIPT" | head -n1 | cut -d: -f1) + + [[ -n "$rm_line" ]] + [[ -n "$mkdir_line" ]] + [[ -n "$plugin_line" ]] + [[ -n "$hostname_line" ]] + (( rm_line < plugin_line )) + (( rm_line < hostname_line )) + (( mkdir_line < plugin_line )) + (( mkdir_line < hostname_line )) +} + +@test "(docker-image) [healthcheck] Dockerfile declares HEALTHCHECK with the expected endpoint" { + [[ -f "$DOCKERFILE" ]] + grep -qE "^HEALTHCHECK" "$DOCKERFILE" + grep -qE "http://127\.0\.0\.1:18080/_dokku/health" "$DOCKERFILE" + grep -qE "\-\-start-period" "$DOCKERFILE" +} From 2c274217deb9003fe0debf62e740436df983d613 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 10 May 2026 21:42:35 -0400 Subject: [PATCH 3/4] chore: run shfmt --- .../etc/runit/runsvdir/default/dokku-restore/finish | 4 ++-- tests/image/healthcheck.sh | 8 ++++---- tests/unit/docker-image-healthcheck.bats | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docker/etc/runit/runsvdir/default/dokku-restore/finish b/docker/etc/runit/runsvdir/default/dokku-restore/finish index 84270a92c..39a01df0b 100755 --- a/docker/etc/runit/runsvdir/default/dokku-restore/finish +++ b/docker/etc/runit/runsvdir/default/dokku-restore/finish @@ -11,8 +11,8 @@ fi echo "Done restoring dokku apps." -while ! (echo > /dev/tcp/127.0.0.1/22) >/dev/null 2>&1; do sleep 1; done -while ! (echo > /dev/tcp/127.0.0.1/80) >/dev/null 2>&1; do sleep 1; done +while ! (echo >/dev/tcp/127.0.0.1/22) >/dev/null 2>&1; do sleep 1; done +while ! (echo >/dev/tcp/127.0.0.1/80) >/dev/null 2>&1; do sleep 1; done touch /var/run/dokku/ready echo "Dokku is ready." diff --git a/tests/image/healthcheck.sh b/tests/image/healthcheck.sh index 49f114e1a..6fb2399b4 100755 --- a/tests/image/healthcheck.sh +++ b/tests/image/healthcheck.sh @@ -38,7 +38,7 @@ start-container() { wait-for-catchall-vhost() { local deadline=$((SECONDS + 30)) - while (( SECONDS < deadline )); do + while ((SECONDS < deadline)); do if docker exec "$CONTAINER_NAME" test -f /etc/nginx/conf.d/00-default-vhost.conf; then return 0 fi @@ -53,7 +53,7 @@ plugins/nginx-vhosts/templates/default-site.conf. Rebuild the deb before running wait-for-healthy() { local deadline=$((SECONDS + START_TIMEOUT_SECONDS)) local status - while (( SECONDS < deadline )); do + while ((SECONDS < deadline)); do status=$(docker inspect --format='{{.State.Health.Status}}' "$CONTAINER_NAME" 2>/dev/null || echo "missing") if [[ "$status" == "healthy" ]]; then log-info "Container reached healthy status" @@ -95,14 +95,14 @@ assert-unhealthy-after-sentinel-removed() { local exit_code=0 docker exec "$CONTAINER_NAME" curl -fsS http://127.0.0.1:18080/_dokku/health >/dev/null 2>&1 || exit_code=$? - if (( exit_code == 0 )); then + if ((exit_code == 0)); then log-fail "Health endpoint still returned success after the readiness sentinel was removed" fi log-info "Waiting for docker to flip the health status to unhealthy" local deadline=$((SECONDS + UNHEALTHY_TIMEOUT_SECONDS)) local status - while (( SECONDS < deadline )); do + while ((SECONDS < deadline)); do status=$(docker inspect --format='{{.State.Health.Status}}' "$CONTAINER_NAME" 2>/dev/null || echo "missing") if [[ "$status" == "unhealthy" ]]; then log-info "Container flipped to unhealthy" diff --git a/tests/unit/docker-image-healthcheck.bats b/tests/unit/docker-image-healthcheck.bats index e91a65829..27d75caea 100644 --- a/tests/unit/docker-image-healthcheck.bats +++ b/tests/unit/docker-image-healthcheck.bats @@ -26,8 +26,8 @@ DOCKERFILE="${REPO_ROOT}/Dockerfile" [[ -n "$sshd_line" ]] [[ -n "$nginx_line" ]] [[ -n "$touch_line" ]] - (( touch_line > sshd_line )) - (( touch_line > nginx_line )) + ((touch_line > sshd_line)) + ((touch_line > nginx_line)) } @test "(docker-image) [healthcheck] my_init clears stale sentinel on container boot" { @@ -43,10 +43,10 @@ DOCKERFILE="${REPO_ROOT}/Dockerfile" [[ -n "$mkdir_line" ]] [[ -n "$plugin_line" ]] [[ -n "$hostname_line" ]] - (( rm_line < plugin_line )) - (( rm_line < hostname_line )) - (( mkdir_line < plugin_line )) - (( mkdir_line < hostname_line )) + ((rm_line < plugin_line)) + ((rm_line < hostname_line)) + ((mkdir_line < plugin_line)) + ((mkdir_line < hostname_line)) } @test "(docker-image) [healthcheck] Dockerfile declares HEALTHCHECK with the expected endpoint" { From 66eb1340f51945cbbf49fcbd0aae9592591f71be Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 10 May 2026 22:54:49 -0400 Subject: [PATCH 4/4] fix: install nginx catch-all default site in dokku container Pre-seeds the `dokku/install_default_site` debconf answer to `true` so the dokku postinst installs `/etc/nginx/conf.d/00-default-vhost.conf` during image build. Without this, debconf returned an empty value in non-interactive docker builds, the postinst's `setup-default-site` short-circuited, and nginx had no listener on port 80 - which left the readiness sentinel untouched and the container stuck unhealthy. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 6dca6ff8e..3b2229b5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ RUN mkdir -p /etc/apt/keyrings \ && echo "dokku dokku/hostname string $DOKKU_HOSTNAME" | debconf-set-selections \ && echo "dokku dokku/skip_key_file boolean $DOKKU_SKIP_KEY_FILE" | debconf-set-selections \ && echo "dokku dokku/vhost_enable boolean $DOKKU_VHOST_ENABLE" | debconf-set-selections \ + && echo "dokku dokku/install_default_site boolean true" | debconf-set-selections \ && curl -sSL https://packagecloud.io/dokku/dokku/gpgkey | apt-key add - \ && echo "deb https://packagecloud.io/dokku/dokku/ubuntu/ noble main" | tee /etc/apt/sources.list.d/dokku.list \ && mkdir -p /etc/nginx/ \