Merge pull request #8669 from dokku/8630-ps-restart-and-other-commands-fail-after-local-image-gc-when-registry-push-on-release-is-enabled
Recover deployed image from registry on local miss
This commit is contained in:
9 files changed
+155
-27
No files matched your search
@@ -178,6 +178,10 @@ dokku registry:set node-js-app push-on-release
|
|||||||
dokku registry:set --global push-on-release
|
dokku registry:set --global push-on-release
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### Recovering from local image GC
|
||||||
|
|
||||||
|
When `push-on-release` is enabled, Dokku treats the remote registry as the canonical store for app images. If a local image disappears - for example because a `docker image prune` cron ran, the host rebooted, or an operator removed it manually - subsequent commands that need the image (`ps:restart`, `ps:scale`, `dokku run`, `domains:add`, `certs:add`, etc.) will pull the missing tag back from the registry automatically. Per-app registry credentials configured via `registry:login` are honored during the pull. This recovery covers the deployed numeric tag pushed by the registry plugin - a missing `latest` tag will be ignored.
|
||||||
|
|
||||||
### Push extra tags
|
### Push extra tags
|
||||||
|
|
||||||
To push the image on release with extra tags, set the `push-extra-tags` to a comma-separated list of tags via the `registry:set` command. The default value for this property is empty. Setting the property will result in the image being tagged with extra tags every release.
|
To push the image on release with extra tags, set the `push-extra-tags` to a comma-separated list of tags via the `registry:set` command. The default value for this property is empty. Setting the property will result in the image being tagged with extra tags every release.
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ cmd-report() {
|
|||||||
dokku_log_info1 "herokuish version: "
|
dokku_log_info1 "herokuish version: "
|
||||||
if [[ "$ARCHITECTURE" == "arm64" ]]; then
|
if [[ "$ARCHITECTURE" == "arm64" ]]; then
|
||||||
dokku_log_warn "herokuish not supported on $ARCHITECTURE architecture"
|
dokku_log_warn "herokuish not supported on $ARCHITECTURE architecture"
|
||||||
elif verify_image "$DOKKU_IMAGE"; then
|
elif fn-verify-app-image "" "$DOKKU_IMAGE"; then
|
||||||
"$DOCKER_BIN" container run $DOKKU_GLOBAL_RUN_ARGS --rm "$DOKKU_IMAGE" herokuish version | sed "s/^/ /"
|
"$DOCKER_BIN" container run $DOKKU_GLOBAL_RUN_ARGS --rm "$DOKKU_IMAGE" herokuish version | sed "s/^/ /"
|
||||||
else
|
else
|
||||||
dokku_log_warn "Herokuish image $DOKKU_IMAGE is not available"
|
dokku_log_warn "Herokuish image $DOKKU_IMAGE is not available"
|
||||||
|
|||||||
+45
-21
@@ -279,8 +279,9 @@ verify_app_name() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
verify_image() {
|
verify_image() {
|
||||||
declare desc="verify image existence"
|
declare desc="[deprecated] verify image existence. Use 'fn-verify-app-image' instead."
|
||||||
local IMAGE="$1"
|
local IMAGE="$1"
|
||||||
|
dokku_log_warn "Deprecated: please use 'fn-verify-app-image' instead of 'verify_image'"
|
||||||
if "$DOCKER_BIN" image inspect "$IMAGE" &>/dev/null; then
|
if "$DOCKER_BIN" image inspect "$IMAGE" &>/dev/null; then
|
||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
@@ -288,6 +289,46 @@ verify_image() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn-verify-app-image() {
|
||||||
|
declare desc="verify image existence, pulling from the registry when push-on-release is enabled"
|
||||||
|
declare APP="$1" IMAGE="$2"
|
||||||
|
|
||||||
|
if "$DOCKER_BIN" image inspect "$IMAGE" &>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$IMAGE" == *:latest ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local DEPLOYED_TAG
|
||||||
|
DEPLOYED_TAG="$(plugn trigger deployed-app-image-tag "$APP")"
|
||||||
|
if [[ -z "$DEPLOYED_TAG" ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local DOCKER_CONFIG
|
||||||
|
DOCKER_CONFIG="$(fn-registry-docker-config-dir "$APP")"
|
||||||
|
[[ -n "$DOCKER_CONFIG" ]] && export DOCKER_CONFIG
|
||||||
|
|
||||||
|
local REMOTE_REPO DEPLOYED_REPO TAG
|
||||||
|
REMOTE_REPO="$(plugn trigger deployed-app-repository "$APP")"
|
||||||
|
DEPLOYED_REPO="$(plugn trigger deployed-app-image-repo "$APP")"
|
||||||
|
TAG="${IMAGE##*:}"
|
||||||
|
|
||||||
|
dokku_log_info1 "Image $IMAGE not found locally, pulling from registry" 1>&2
|
||||||
|
if [[ -n "$DEPLOYED_REPO" ]] && [[ "$IMAGE" == "${REMOTE_REPO}${DEPLOYED_REPO}:"* ]]; then
|
||||||
|
"$DOCKER_BIN" image pull "$IMAGE" 1>&2 && return 0
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local REMOTE_IMAGE="${REMOTE_REPO}${DEPLOYED_REPO:-${IMAGE%:*}}:${TAG}"
|
||||||
|
if "$DOCKER_BIN" image pull "$REMOTE_IMAGE" 1>&2; then
|
||||||
|
"$DOCKER_BIN" image tag "$REMOTE_IMAGE" "$IMAGE" 1>&2 && return 0
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
get_app_image_repo() {
|
get_app_image_repo() {
|
||||||
declare desc="central definition of image repo pattern"
|
declare desc="central definition of image repo pattern"
|
||||||
local APP="$1"
|
local APP="$1"
|
||||||
@@ -309,7 +350,7 @@ get_deploying_app_image_name() {
|
|||||||
[[ -z "$IMAGE_REPO" ]] && IMAGE_REPO=$(get_app_image_repo "$APP")
|
[[ -z "$IMAGE_REPO" ]] && IMAGE_REPO=$(get_app_image_repo "$APP")
|
||||||
|
|
||||||
local IMAGE="${IMAGE_REMOTE_REPOSITORY}${IMAGE_REPO}:${IMAGE_TAG}"
|
local IMAGE="${IMAGE_REMOTE_REPOSITORY}${IMAGE_REPO}:${IMAGE_TAG}"
|
||||||
verify_image "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found"
|
fn-verify-app-image "$APP" "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found"
|
||||||
echo "$IMAGE"
|
echo "$IMAGE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -325,7 +366,7 @@ get_app_image_name() {
|
|||||||
|
|
||||||
if [[ -n "$IMAGE_TAG" ]]; then
|
if [[ -n "$IMAGE_TAG" ]]; then
|
||||||
local IMAGE="$IMAGE_REPO:$IMAGE_TAG"
|
local IMAGE="$IMAGE_REPO:$IMAGE_TAG"
|
||||||
verify_image "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found"
|
fn-verify-app-image "$APP" "$IMAGE" || dokku_log_fail "App image ($IMAGE) not found"
|
||||||
else
|
else
|
||||||
local IMAGE="$IMAGE_REPO:latest"
|
local IMAGE="$IMAGE_REPO:latest"
|
||||||
fi
|
fi
|
||||||
@@ -627,7 +668,7 @@ release_and_deploy() {
|
|||||||
local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
|
local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")
|
||||||
local exit_code=0
|
local exit_code=0
|
||||||
|
|
||||||
if verify_image "$IMAGE"; then
|
if fn-verify-app-image "$APP" "$IMAGE"; then
|
||||||
if ! declare -f -F fn-plugin-property-get-default >/dev/null; then
|
if ! declare -f -F fn-plugin-property-get-default >/dev/null; then
|
||||||
source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
|
||||||
fi
|
fi
|
||||||
@@ -752,23 +793,6 @@ get_ipv6_regex() {
|
|||||||
echo "${RE_IPV6}\$"
|
echo "${RE_IPV6}\$"
|
||||||
}
|
}
|
||||||
|
|
||||||
get_entrypoint_from_image() {
|
|
||||||
declare desc="return .Config.Entrypoint from passed image name"
|
|
||||||
local IMAGE="$1"
|
|
||||||
verify_image "$IMAGE"
|
|
||||||
local DOCKER_IMAGE_ENTRYPOINT="$("$DOCKER_BIN" image inspect --format '{{range .Config.Entrypoint}}{{.}} {{end}}' "$IMAGE")"
|
|
||||||
echo "ENTRYPOINT $DOCKER_IMAGE_ENTRYPOINT"
|
|
||||||
}
|
|
||||||
|
|
||||||
get_cmd_from_image() {
|
|
||||||
declare desc="return .Config.Cmd from passed image name"
|
|
||||||
local IMAGE="$1"
|
|
||||||
verify_image "$IMAGE"
|
|
||||||
local DOCKER_IMAGE_CMD="$("$DOCKER_BIN" image inspect --format '{{range .Config.Cmd}}{{.}} {{end}}' "$IMAGE")"
|
|
||||||
DOCKER_IMAGE_CMD="${DOCKER_IMAGE_CMD/\/bin\/sh -c/}"
|
|
||||||
echo "CMD $DOCKER_IMAGE_CMD"
|
|
||||||
}
|
|
||||||
|
|
||||||
extract_directive_from_dockerfile() {
|
extract_directive_from_dockerfile() {
|
||||||
declare desc="return requested directive from passed file path"
|
declare desc="return requested directive from passed file path"
|
||||||
local FILE_PATH="$1"
|
local FILE_PATH="$1"
|
||||||
|
|||||||
@@ -176,7 +176,7 @@ cmd-git-load-image() {
|
|||||||
dokku_setup_build_capture "$APP" "git:load-image"
|
dokku_setup_build_capture "$APP" "git:load-image"
|
||||||
cat | docker load
|
cat | docker load
|
||||||
|
|
||||||
if ! verify_image "$DOCKER_IMAGE"; then
|
if ! fn-verify-app-image "$APP" "$DOCKER_IMAGE"; then
|
||||||
dokku_log_fail "Loaded image tarball but the specified docker image was not found: $DOCKER_IMAGE"
|
dokku_log_fail "Loaded image tarball but the specified docker image was not found: $DOCKER_IMAGE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -510,7 +510,7 @@ nginx_build_config() {
|
|||||||
if [[ "$(plugn trigger network-get-static-listeners "$APP" "web")" == "" ]]; then
|
if [[ "$(plugn trigger network-get-static-listeners "$APP" "web")" == "" ]]; then
|
||||||
local IMAGE_TAG=$(get_running_image_tag "$APP")
|
local IMAGE_TAG=$(get_running_image_tag "$APP")
|
||||||
local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG" 2>/dev/null)
|
local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG" 2>/dev/null)
|
||||||
if ! verify_image "$IMAGE" 2>/dev/null; then
|
if ! fn-verify-app-image "$APP" "$IMAGE" 2>/dev/null; then
|
||||||
dokku_log_fail "Missing image for app"
|
dokku_log_fail "Missing image for app"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func main() {
|
|||||||
chown := args.String("chown", "herokuish", "--chown: chown option (herokuish, heroku, paketo, root, false)")
|
chown := args.String("chown", "herokuish", "--chown: chown option (herokuish, heroku, paketo, root, false)")
|
||||||
args.Parse(os.Args[2:])
|
args.Parse(os.Args[2:])
|
||||||
directory := args.Arg(0)
|
directory := args.Arg(0)
|
||||||
common.LogWarn("storage:ensure-directory is deprecated; use storage:create instead.")
|
common.LogWarn("Deprecated: please use 'storage:create' instead of 'storage:ensure-directory'")
|
||||||
err = storage.CommandEnsureDirectory(directory, *chown)
|
err = storage.CommandEnsureDirectory(directory, *chown)
|
||||||
case "info":
|
case "info":
|
||||||
args := flag.NewFlagSet("storage:info", flag.ExitOnError)
|
args := flag.NewFlagSet("storage:info", flag.ExitOnError)
|
||||||
|
|||||||
@@ -180,7 +180,7 @@ func detectDistro() string {
|
|||||||
// should use storage.ListAppMountEntries directly. A deprecation warning
|
// should use storage.ListAppMountEntries directly. A deprecation warning
|
||||||
// is emitted on every invocation.
|
// is emitted on every invocation.
|
||||||
func TriggerStorageList(appName string, phase string, format string) error {
|
func TriggerStorageList(appName string, phase string, format string) error {
|
||||||
common.LogWarn("the storage-list plugn trigger is deprecated; use the storage-app-mounts trigger or the storage Go package directly")
|
common.LogWarn("Deprecated: please use the 'storage-app-mounts' plugn trigger or the storage Go package directly instead of 'storage-list'")
|
||||||
|
|
||||||
rows, err := ListAppMountEntries(appName, phase)
|
rows, err := ListAppMountEntries(appName, phase)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
#!/usr/bin/env bats
|
||||||
|
|
||||||
|
load test_helper
|
||||||
|
|
||||||
|
setup() {
|
||||||
|
global_setup
|
||||||
|
create_app
|
||||||
|
dokku checks:set $TEST_APP wait-to-retire 30
|
||||||
|
}
|
||||||
|
|
||||||
|
teardown() {
|
||||||
|
destroy_app
|
||||||
|
global_teardown
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_push_on_release_app() {
|
||||||
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||||
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku builder:set $TEST_APP selected herokuish"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku proxy:set $TEST_APP nginx"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku registry:login $TEST_APP docker.io $DOCKERHUB_USERNAME $DOCKERHUB_TOKEN"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku registry:set $TEST_APP image-repo $DOCKERHUB_USERNAME/$TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku registry:set $TEST_APP push-on-release true"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "docker image rm $DOCKERHUB_USERNAME/$TEST_APP:1"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "docker image inspect $DOCKERHUB_USERNAME/$TEST_APP:1"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_failure
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "(registry) [recover] ps:restart pulls missing image" {
|
||||||
|
setup_push_on_release_app
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku ps:restart $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "not found locally, pulling from registry"
|
||||||
|
|
||||||
|
run /bin/bash -c "docker image inspect $DOCKERHUB_USERNAME/$TEST_APP:1"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "(registry) [recover] dokku run pulls missing image" {
|
||||||
|
setup_push_on_release_app
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku run $TEST_APP echo recovery-ok"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output_contains "not found locally, pulling from registry"
|
||||||
|
assert_output_contains "recovery-ok"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "(registry) [recover] domains:add pulls missing image" {
|
||||||
|
setup_push_on_release_app
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku domains:add $TEST_APP recovery.${DOKKU_DOMAIN}"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "docker image inspect $DOCKERHUB_USERNAME/$TEST_APP:1"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
}
|
||||||
@@ -443,7 +443,7 @@ teardown() {
|
|||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
assert_output_contains "deprecated"
|
assert_output_contains "Deprecated:"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "(storage:migrate) leaves legacy-mounts-migrated unset for apps with no legacy -v lines" {
|
@test "(storage:migrate) leaves legacy-mounts-migrated unset for apps with no legacy -v lines" {
|
||||||
|
|||||||
Reference in new issue
Block a user