Merge pull request #8669 from dokku/8630-ps-restart-and-other-commands-fail-after-local-image-gc-when-registry-push-on-release-is-enabled

Recover deployed image from registry on local miss
This commit is contained in:
Jose Diaz-Gonzalez authored and GitHub committed 2026-05-26 21:09:15 -04:00
commit f9186ca1fd
9 files changed
+155 -27

No files matched your search

@@ -178,6 +178,10 @@ dokku registry:set node-js-app push-on-release
dokku registry:set --global push-on-release dokku registry:set --global push-on-release
``` ```
#### Recovering from local image GC
When `push-on-release` is enabled, Dokku treats the remote registry as the canonical store for app images. If a local image disappears - for example because a `docker image prune` cron ran, the host rebooted, or an operator removed it manually - subsequent commands that need the image (`ps:restart`, `ps:scale`, `dokku run`, `domains:add`, `certs:add`, etc.) will pull the missing tag back from the registry automatically. Per-app registry credentials configured via `registry:login` are honored during the pull. This recovery covers the deployed numeric tag pushed by the registry plugin - a missing `latest` tag will be ignored.
### Push extra tags ### Push extra tags
To push the image on release with extra tags, set the `push-extra-tags` to a comma-separated list of tags via the `registry:set` command. The default value for this property is empty. Setting the property will result in the image being tagged with extra tags every release. To push the image on release with extra tags, set the `push-extra-tags` to a comma-separated list of tags via the `registry:set` command. The default value for this property is empty. Setting the property will result in the image being tagged with extra tags every release.
+1 -1
View File
@@ -28,7 +28,7 @@ cmd-report() {
dokku_log_info1 "herokuish version: " dokku_log_info1 "herokuish version: "
if [[ "$ARCHITECTURE" == "arm64" ]]; then if [[ "$ARCHITECTURE" == "arm64" ]]; then
dokku_log_warn "herokuish not supported on $ARCHITECTURE architecture" dokku_log_warn "herokuish not supported on $ARCHITECTURE architecture"
elif verify_image "$DOKKU_IMAGE"; then elif fn-verify-app-image "" "$DOKKU_IMAGE"; then
"$DOCKER_BIN" container run $DOKKU_GLOBAL_RUN_ARGS --rm "$DOKKU_IMAGE" herokuish version | sed "s/^/ /" "$DOCKER_BIN" container run $DOKKU_GLOBAL_RUN_ARGS --rm "$DOKKU_IMAGE" herokuish version | sed "s/^/ /"
else else
dokku_log_warn "Herokuish image $DOKKU_IMAGE is not available" dokku_log_warn "Herokuish image $DOKKU_IMAGE is not available"
+45 -21
View File
@@ -279,8 +279,9 @@ verify_app_name() {
} }
verify_image() { verify_image() {
declare desc="verify image existence" declare desc="[deprecated] verify image existence. Use 'fn-verify-app-image' instead."
local IMAGE="$1" local IMAGE="$1"
dokku_log_warn "Deprecated: please use 'fn-verify-app-image' instead of 'verify_image'"
if "$DOCKER_BIN" image inspect "$IMAGE" &>/dev/null; then if "$DOCKER_BIN" image inspect "$IMAGE" &>/dev/null; then
return 0 return 0
else else
@@ -288,6 +289,46 @@ verify_image() {
fi fi
} }
fn-verify-app-image() {
declare desc="verify image existence, pulling from the registry when push-on-release is enabled"
declare APP="$1" IMAGE="$2"
if "$DOCKER_BIN" image inspect "$IMAGE" &>/dev/null; then
return 0
fi
if [[ "$IMAGE" == *:latest ]]; then
return 1
fi
local DEPLOYED_TAG
DEPLOYED_TAG="$(plugn trigger deployed-app-image-tag "$APP")"
if [[ -z "$DEPLOYED_TAG" ]]; then
return 1
fi
local DOCKER_CONFIG
DOCKER_CONFIG="$(fn-registry-docker-config-dir "$APP")"
[[ -n "$DOCKER_CONFIG" ]] && export DOCKER_CONFIG
local REMOTE_REPO DEPLOYED_REPO TAG
REMOTE_REPO="$(plugn trigger deployed-app-repository "$APP")"
DEPLOYED_REPO="$(plugn trigger deployed-app-image-repo "$APP")"
TAG="${IMAGE##*:}"
dokku_log_info1 "Image $IMAGE not found locally, pulling from registry" 1>&2
if [[ -n "$DEPLOYED_REPO" ]] && [[ "$IMAGE" == "${REMOTE_REPO}${DEPLOYED_REPO}:"* ]]; then
"$DOCKER_BIN" image pull "$IMAGE" 1>&2 && return 0
return 1
fi
local REMOTE_IMAGE="${REMOTE_REPO}${DEPLOYED_REPO:-${IMAGE%:*}}:${TAG}"
if "$DOCKER_BIN" image pull "$REMOTE_IMAGE" 1>&2; then
"$DOCKER_BIN" image tag "$REMOTE_IMAGE" "$IMAGE" 1>&2 && return 0
fi
return 1
}
get_app_image_repo() { get_app_image_repo() {
declare desc="central definition of image repo pattern" declare desc="central definition of image repo pattern"
local APP="$1" local APP="$1"
@@ -309,7 +350,7 @@ get_deploying_app_image_name() {
[[ -z "$IMAGE_REPO" ]] && IMAGE_REPO=$(get_app_image_repo "$APP") [[ -z "$IMAGE_REPO" ]] && IMAGE_REPO=$(get_app_image_repo "$APP")
local IMAGE="${IMAGE_REMOTE_REPOSITORY}${IMAGE_REPO}:${IMAGE_TAG}" local IMAGE="${IMAGE_REMOTE_REPOSITORY}${IMAGE_REPO}:${IMAGE_TAG}"
verify_image "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found" fn-verify-app-image "$APP" "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found"
echo "$IMAGE" echo "$IMAGE"
} }
@@ -325,7 +366,7 @@ get_app_image_name() {
if [[ -n "$IMAGE_TAG" ]]; then if [[ -n "$IMAGE_TAG" ]]; then
local IMAGE="$IMAGE_REPO:$IMAGE_TAG" local IMAGE="$IMAGE_REPO:$IMAGE_TAG"
verify_image "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found" fn-verify-app-image "$APP" "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found"
else else
local IMAGE="$IMAGE_REPO:latest" local IMAGE="$IMAGE_REPO:latest"
fi fi
@@ -627,7 +668,7 @@ release_and_deploy() {
local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
local exit_code=0 local exit_code=0
if verify_image "$IMAGE"; then if fn-verify-app-image "$APP" "$IMAGE"; then
if ! declare -f -F fn-plugin-property-get-default >/dev/null; then if ! declare -f -F fn-plugin-property-get-default >/dev/null; then
source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions" source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
fi fi
@@ -752,23 +793,6 @@ get_ipv6_regex() {
echo "${RE_IPV6}\$" echo "${RE_IPV6}\$"
} }
get_entrypoint_from_image() {
declare desc="return .Config.Entrypoint from passed image name"
local IMAGE="$1"
verify_image "$IMAGE"
local DOCKER_IMAGE_ENTRYPOINT="$("$DOCKER_BIN" image inspect --format '{{range .Config.Entrypoint}}{{.}} {{end}}' "$IMAGE")"
echo "ENTRYPOINT $DOCKER_IMAGE_ENTRYPOINT"
}
get_cmd_from_image() {
declare desc="return .Config.Cmd from passed image name"
local IMAGE="$1"
verify_image "$IMAGE"
local DOCKER_IMAGE_CMD="$("$DOCKER_BIN" image inspect --format '{{range .Config.Cmd}}{{.}} {{end}}' "$IMAGE")"
DOCKER_IMAGE_CMD="${DOCKER_IMAGE_CMD/\/bin\/sh -c/}"
echo "CMD $DOCKER_IMAGE_CMD"
}
extract_directive_from_dockerfile() { extract_directive_from_dockerfile() {
declare desc="return requested directive from passed file path" declare desc="return requested directive from passed file path"
local FILE_PATH="$1" local FILE_PATH="$1"
+1 -1
View File
@@ -176,7 +176,7 @@ cmd-git-load-image() {
dokku_setup_build_capture "$APP" "git:load-image" dokku_setup_build_capture "$APP" "git:load-image"
cat | docker load cat | docker load
if ! verify_image "$DOCKER_IMAGE"; then if ! fn-verify-app-image "$APP" "$DOCKER_IMAGE"; then
dokku_log_fail "Loaded image tarball but the specified docker image was not found: $DOCKER_IMAGE" dokku_log_fail "Loaded image tarball but the specified docker image was not found: $DOCKER_IMAGE"
fi fi
+1 -1
View File
@@ -510,7 +510,7 @@ nginx_build_config() {
if [[ "$(plugn trigger network-get-static-listeners "$APP" "web")" == "" ]]; then if [[ "$(plugn trigger network-get-static-listeners "$APP" "web")" == "" ]]; then
local IMAGE_TAG=$(get_running_image_tag "$APP") local IMAGE_TAG=$(get_running_image_tag "$APP")
local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG" 2>/dev/null) local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG" 2>/dev/null)
if ! verify_image "$IMAGE" 2>/dev/null; then if ! fn-verify-app-image "$APP" "$IMAGE" 2>/dev/null; then
dokku_log_fail "Missing image for app" dokku_log_fail "Missing image for app"
fi fi
fi fi
@@ -65,7 +65,7 @@ func main() {
chown := args.String("chown", "herokuish", "--chown: chown option (herokuish, heroku, paketo, root, false)") chown := args.String("chown", "herokuish", "--chown: chown option (herokuish, heroku, paketo, root, false)")
args.Parse(os.Args[2:]) args.Parse(os.Args[2:])
directory := args.Arg(0) directory := args.Arg(0)
common.LogWarn("storage:ensure-directory is deprecated; use storage:create instead.") common.LogWarn("Deprecated: please use 'storage:create' instead of 'storage:ensure-directory'")
err = storage.CommandEnsureDirectory(directory, *chown) err = storage.CommandEnsureDirectory(directory, *chown)
case "info": case "info":
args := flag.NewFlagSet("storage:info", flag.ExitOnError) args := flag.NewFlagSet("storage:info", flag.ExitOnError)
+1 -1
View File
@@ -180,7 +180,7 @@ func detectDistro() string {
// should use storage.ListAppMountEntries directly. A deprecation warning // should use storage.ListAppMountEntries directly. A deprecation warning
// is emitted on every invocation. // is emitted on every invocation.
func TriggerStorageList(appName string, phase string, format string) error { func TriggerStorageList(appName string, phase string, format string) error {
common.LogWarn("the storage-list plugn trigger is deprecated; use the storage-app-mounts trigger or the storage Go package directly") common.LogWarn("Deprecated: please use the 'storage-app-mounts' plugn trigger or the storage Go package directly instead of 'storage-list'")
rows, err := ListAppMountEntries(appName, phase) rows, err := ListAppMountEntries(appName, phase)
if err != nil { if err != nil {
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env bats
load test_helper
setup() {
global_setup
create_app
dokku checks:set $TEST_APP wait-to-retire 30
}
teardown() {
destroy_app
global_teardown
}
setup_push_on_release_app() {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi
run /bin/bash -c "dokku builder:set $TEST_APP selected herokuish"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku proxy:set $TEST_APP nginx"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku registry:login $TEST_APP docker.io $DOCKERHUB_USERNAME $DOCKERHUB_TOKEN"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku registry:set $TEST_APP image-repo $DOCKERHUB_USERNAME/$TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku registry:set $TEST_APP push-on-release true"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker image rm $DOCKERHUB_USERNAME/$TEST_APP:1"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker image inspect $DOCKERHUB_USERNAME/$TEST_APP:1"
echo "output: $output"
echo "status: $status"
assert_failure
}
@test "(registry) [recover] ps:restart pulls missing image" {
setup_push_on_release_app
run /bin/bash -c "dokku ps:restart $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "not found locally, pulling from registry"
run /bin/bash -c "docker image inspect $DOCKERHUB_USERNAME/$TEST_APP:1"
echo "output: $output"
echo "status: $status"
assert_success
}
@test "(registry) [recover] dokku run pulls missing image" {
setup_push_on_release_app
run /bin/bash -c "dokku run $TEST_APP echo recovery-ok"
echo "output: $output"
echo "status: $status"
assert_success
assert_output_contains "not found locally, pulling from registry"
assert_output_contains "recovery-ok"
}
@test "(registry) [recover] domains:add pulls missing image" {
setup_push_on_release_app
run /bin/bash -c "dokku domains:add $TEST_APP recovery.${DOKKU_DOMAIN}"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "docker image inspect $DOCKERHUB_USERNAME/$TEST_APP:1"
echo "output: $output"
echo "status: $status"
assert_success
}
+1 -1
View File
@@ -443,7 +443,7 @@ teardown() {
echo "output: $output" echo "output: $output"
echo "status: $status" echo "status: $status"
assert_success assert_success
assert_output_contains "deprecated" assert_output_contains "Deprecated:"
} }
@test "(storage:migrate) leaves legacy-mounts-migrated unset for apps with no legacy -v lines" { @test "(storage:migrate) leaves legacy-mounts-migrated unset for apps with no legacy -v lines" {