Merge pull request #4202 from dokku/4104-aggressive-dangling-clean
Schedule related images for cleanup when retiring containers
This commit is contained in:
@@ -1914,7 +1914,7 @@ DOKKU_SCHEDULER="$1"; APP="$2"; CONTAINER_ID="$3";
|
|||||||
> Warning: The scheduler plugin trigger apis are under development and may change
|
> Warning: The scheduler plugin trigger apis are under development and may change
|
||||||
> between minor releases until the 1.0 release.
|
> between minor releases until the 1.0 release.
|
||||||
|
|
||||||
- Description: Allows scheduling retiring a local container
|
- Description: Allows scheduling retiring a local container and any related images
|
||||||
- Invoked by: `internally`
|
- Invoked by: `internally`
|
||||||
- Arguments: `$APP $CONTAINER_ID`
|
- Arguments: `$APP $CONTAINER_ID`
|
||||||
- Example:
|
- Example:
|
||||||
|
|||||||
@@ -30,10 +30,18 @@ func main() {
|
|||||||
oldAppName := flag.Arg(0)
|
oldAppName := flag.Arg(0)
|
||||||
newAppName := flag.Arg(1)
|
newAppName := flag.Arg(1)
|
||||||
err = ps.TriggerPostAppClone(oldAppName, newAppName)
|
err = ps.TriggerPostAppClone(oldAppName, newAppName)
|
||||||
|
case "post-app-clone-setup":
|
||||||
|
oldAppName := flag.Arg(0)
|
||||||
|
newAppName := flag.Arg(1)
|
||||||
|
err = ps.TriggerPostAppCloneSetup(oldAppName, newAppName)
|
||||||
case "post-app-rename":
|
case "post-app-rename":
|
||||||
oldAppName := flag.Arg(0)
|
oldAppName := flag.Arg(0)
|
||||||
newAppName := flag.Arg(1)
|
newAppName := flag.Arg(1)
|
||||||
err = ps.TriggerPostAppRename(oldAppName, newAppName)
|
err = ps.TriggerPostAppRename(oldAppName, newAppName)
|
||||||
|
case "post-app-rename-setup":
|
||||||
|
oldAppName := flag.Arg(0)
|
||||||
|
newAppName := flag.Arg(1)
|
||||||
|
err = ps.TriggerPostAppRenameSetup(oldAppName, newAppName)
|
||||||
case "post-create":
|
case "post-create":
|
||||||
appName := flag.Arg(0)
|
appName := flag.Arg(0)
|
||||||
err = ps.TriggerPostCreate(appName)
|
err = ps.TriggerPostCreate(appName)
|
||||||
|
|||||||
@@ -31,8 +31,8 @@ func TriggerCorePostDeploy(appName string) error {
|
|||||||
|
|
||||||
// TriggerInstall initializes app restart policies
|
// TriggerInstall initializes app restart policies
|
||||||
func TriggerInstall() error {
|
func TriggerInstall() error {
|
||||||
if err := common.PropertySetup("buildpacks"); err != nil {
|
if err := common.PropertySetup("ps"); err != nil {
|
||||||
return fmt.Errorf("Unable to install the buildpacks plugin: %s", err.Error())
|
return fmt.Errorf("Unable to install the ps plugin: %s", err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps")
|
directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "ps")
|
||||||
@@ -76,6 +76,16 @@ func TriggerPostAppClone(oldAppName string, newAppName string) error {
|
|||||||
return Rebuild(newAppName)
|
return Rebuild(newAppName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TriggerPostAppCloneSetup creates new ps files
|
||||||
|
func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
|
||||||
|
err := common.PropertyClone("ps", oldAppName, newAppName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// TriggerPostAppRename rebuilds the renamed app
|
// TriggerPostAppRename rebuilds the renamed app
|
||||||
func TriggerPostAppRename(oldAppName string, newAppName string) error {
|
func TriggerPostAppRename(oldAppName string, newAppName string) error {
|
||||||
if os.Getenv("SKIP_REBUILD") == "true" {
|
if os.Getenv("SKIP_REBUILD") == "true" {
|
||||||
@@ -85,6 +95,19 @@ func TriggerPostAppRename(oldAppName string, newAppName string) error {
|
|||||||
return Rebuild(newAppName)
|
return Rebuild(newAppName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TriggerPostAppRenameSetup renames ps files
|
||||||
|
func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
|
||||||
|
if err := common.PropertyClone("ps", oldAppName, newAppName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := common.PropertyDestroy("ps", oldAppName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// TriggerPostCreate ensures apps have a default restart policy
|
// TriggerPostCreate ensures apps have a default restart policy
|
||||||
// and scale value for web
|
// and scale value for web
|
||||||
func TriggerPostCreate(appName string) error {
|
func TriggerPostCreate(appName string) error {
|
||||||
|
|||||||
@@ -105,12 +105,116 @@ fn-scheduler-docker-local-retire-container() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
fn-scheduler-docker-local-register-retired-container() {
|
fn-scheduler-docker-local-retire-containers() {
|
||||||
declare APP="$1" CID="$2" WAIT="$3"
|
|
||||||
local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
|
local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
|
||||||
|
local APP CID CURRENT_TIME DEAD_TIME STATE
|
||||||
|
|
||||||
|
if [[ ! -f "$DEAD_CONTAINER_FILE" ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
DEAD_CONTAINERS=()
|
||||||
|
while read line; do
|
||||||
|
[[ -z "$line" ]] && continue
|
||||||
|
CURRENT_TIME="$(date +%s)"
|
||||||
|
APP="$(echo "$line" | cut -d ' ' -f1)"
|
||||||
|
CID="$(echo "$line" | cut -d ' ' -f2)"
|
||||||
|
DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
|
||||||
|
|
||||||
|
if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
|
||||||
|
STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
|
||||||
|
if [[ -z "$STATE" ]]; then
|
||||||
|
DEAD_CONTAINERS+=("$CID")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$STATE" == "running" ]]; then
|
||||||
|
dokku_log_warn "Container ${CID} still running"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
"$DOCKER_BIN" container rm --force "$CID" >/dev/null 2>&1 || true
|
||||||
|
if "$DOCKER_BIN" container inspect "${CID}" >/dev/null 2>&1; then
|
||||||
|
dokku_log_warn "Container ${CID} still running"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
DEAD_CONTAINERS+=("$CID")
|
||||||
|
done <"$DEAD_CONTAINER_FILE"
|
||||||
|
|
||||||
|
for CID in "${DEAD_CONTAINERS[@]}"; do
|
||||||
|
sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
fn-scheduler-docker-local-retire-images() {
|
||||||
|
local DEAD_IMAGE_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
|
||||||
|
local APP IMAGE_ID CURRENT_TIME DEAD_TIME STATE RM_OUTPUT
|
||||||
|
|
||||||
|
if [[ ! -f "$DEAD_IMAGE_FILE" ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
DEAD_IMAGES=()
|
||||||
|
while read line; do
|
||||||
|
[[ -z "$line" ]] && continue
|
||||||
|
CURRENT_TIME="$(date +%s)"
|
||||||
|
APP="$(echo "$line" | cut -d ' ' -f1)"
|
||||||
|
IMAGE_ID="$(echo "$line" | cut -d ' ' -f2)"
|
||||||
|
DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
|
||||||
|
|
||||||
|
if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
STATE="$("$DOCKER_BIN" image inspect --format "{{ .Id }}" "$IMAGE_ID" 2>/dev/null || true)"
|
||||||
|
if [[ -z "$STATE" ]]; then
|
||||||
|
DEAD_IMAGES+=("$IMAGE_ID")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
dokku_log_info1_quiet "Attempting to retire $APP image $IMAGE_ID"
|
||||||
|
if RM_OUTPUT="$("$DOCKER_BIN" image rm "$IMAGE_ID" 2>&1)"; then
|
||||||
|
DEAD_IMAGES+=("$IMAGE_ID")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if echo "$RM_OUTPUT" | grep -q "image has dependent child images"; then
|
||||||
|
dokku_log_warn "Image ${IMAGE_ID} has children images, skipping rm and marking dead"
|
||||||
|
DEAD_IMAGES+=("$IMAGE_ID")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if echo "$RM_OUTPUT" | grep -q "image is being used by running container"; then
|
||||||
|
dokku_log_warn "Image ${IMAGE_ID} has running containers, skipping rm"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
dokku_log_warn "Image ${IMAGE_ID} still running"
|
||||||
|
done <"$DEAD_IMAGE_FILE"
|
||||||
|
|
||||||
|
for IMAGE_ID in "${DEAD_IMAGES[@]}"; do
|
||||||
|
sed -i "/${IMAGE_ID}/d" "$DEAD_IMAGE_FILE"
|
||||||
|
done
|
||||||
|
|
||||||
|
sort -o "$DEAD_IMAGE_FILE" -r "$DEAD_IMAGE_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn-scheduler-docker-local-register-retired() {
|
||||||
|
declare TYPE="$1" APP="$2" DOCKER_ID="$3" WAIT="$4"
|
||||||
|
local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
|
||||||
|
if [[ "$TYPE" == "image" ]]; then
|
||||||
|
local DEAD_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-images"
|
||||||
|
fi
|
||||||
local CURRENT_TIME DEAD_TIME
|
local CURRENT_TIME DEAD_TIME
|
||||||
|
|
||||||
CURRENT_TIME="$(date +%s)"
|
CURRENT_TIME="$(date +%s)"
|
||||||
DEAD_TIME=$((CURRENT_TIME + WAIT))
|
DEAD_TIME=$((CURRENT_TIME + WAIT))
|
||||||
echo "${APP} ${CID} ${DEAD_TIME}" >>"${DEAD_CONTAINER_FILE}"
|
if ! grep -q "$DOCKER_ID" "$DEAD_FILE"; then
|
||||||
|
echo "${APP} ${DOCKER_ID} ${DEAD_TIME}" >>"${DEAD_FILE}"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -180,7 +180,7 @@ trigger-scheduler-docker-local-scheduler-deploy() {
|
|||||||
local oldid
|
local oldid
|
||||||
for oldid in $oldids; do
|
for oldid in $oldids; do
|
||||||
dokku_log_verbose_quiet "$oldid"
|
dokku_log_verbose_quiet "$oldid"
|
||||||
fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT"
|
plugn trigger scheduler-register-retired "$APP" "$oldid" "$WAIT"
|
||||||
done
|
done
|
||||||
(
|
(
|
||||||
exec >/dev/null 2>/dev/null </dev/null
|
exec >/dev/null 2>/dev/null </dev/null
|
||||||
|
|||||||
@@ -7,8 +7,14 @@ trigger-scheduler-docker-local-scheduler-register-retired() {
|
|||||||
declare desc="register a container for retiring"
|
declare desc="register a container for retiring"
|
||||||
declare trigger="scheduler-register-retired"
|
declare trigger="scheduler-register-retired"
|
||||||
declare APP="$1" CONTAINER_ID="$2" WAIT="${3:-60}"
|
declare APP="$1" CONTAINER_ID="$2" WAIT="${3:-60}"
|
||||||
|
local IMAGE_ID
|
||||||
|
|
||||||
fn-scheduler-docker-local-register-retired-container "$APP" "$CONTAINER_ID" "$WAIT"
|
IMAGE_ID="$("$DOCKER_BIN" container inspect "$CONTAINER_ID" --format "{{.Image}}" 2>/dev/null | cut -d: -f2 || true)"
|
||||||
|
fn-scheduler-docker-local-register-retired "container" "$APP" "$CONTAINER_ID" "$WAIT"
|
||||||
|
|
||||||
|
if [[ -z "$IMAGE_ID" ]]; then
|
||||||
|
fn-scheduler-docker-local-register-retired "image" "$APP" "$IMAGE_ID" "$WAIT"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
trigger-scheduler-docker-local-scheduler-register-retired "$@"
|
trigger-scheduler-docker-local-scheduler-register-retired "$@"
|
||||||
|
|||||||
@@ -8,48 +8,8 @@ trigger-scheduler-docker-local-scheduler-retire() {
|
|||||||
declare desc="retires all old containers once they have aged out"
|
declare desc="retires all old containers once they have aged out"
|
||||||
declare trigger="scheduler-retire"
|
declare trigger="scheduler-retire"
|
||||||
|
|
||||||
local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
|
fn-scheduler-docker-local-retire-containers
|
||||||
local APP CID CURRENT_TIME DEAD_TIME STATE
|
fn-scheduler-docker-local-retire-images
|
||||||
|
|
||||||
if [[ ! -f "$DEAD_CONTAINER_FILE" ]]; then
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
DEAD_CONTAINERS=()
|
|
||||||
while read line; do
|
|
||||||
CURRENT_TIME="$(date +%s)"
|
|
||||||
APP="$(echo "$line" | cut -d ' ' -f1)"
|
|
||||||
CID="$(echo "$line" | cut -d ' ' -f2)"
|
|
||||||
DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
|
|
||||||
|
|
||||||
if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
|
|
||||||
STATE="$("$DOCKER_BIN" container inspect --format "{{ .State.Status }}" "$CID" 2>/dev/null || true)"
|
|
||||||
if [[ -z "$STATE" ]]; then
|
|
||||||
DEAD_CONTAINERS+=("$CID")
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$STATE" == "running" ]]; then
|
|
||||||
dokku_log_warn "Container ${CID} still running"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
"$DOCKER_BIN" container rm --force "$CID" >/dev/null 2>&1 || true
|
|
||||||
if "$DOCKER_BIN" container inspect "${CID}" >/dev/null 2>&1; then
|
|
||||||
dokku_log_warn "Container ${CID} still running"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
DEAD_CONTAINERS+=("$CID")
|
|
||||||
done <"$DEAD_CONTAINER_FILE"
|
|
||||||
|
|
||||||
for CID in "${DEAD_CONTAINERS[@]}"; do
|
|
||||||
sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
|
|
||||||
done
|
|
||||||
}
|
}
|
||||||
|
|
||||||
trigger-scheduler-docker-local-scheduler-retire "$@"
|
trigger-scheduler-docker-local-scheduler-retire "$@"
|
||||||
|
|||||||
Reference in New Issue
Block a user