From 9e27b07772d78edb6843de6df369dd76f9d656eb Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 4 Oct 2021 03:09:05 -0400 Subject: [PATCH] refactor: split out docker-local scheduler deploys into separate binaries This will allow those binaries to run in parallel. --- plugins/scheduler-docker-local/check-deploy | 30 ++-- .../scheduler-docker-local/scheduler-deploy | 152 ++---------------- 2 files changed, 26 insertions(+), 156 deletions(-) diff --git a/plugins/scheduler-docker-local/check-deploy b/plugins/scheduler-docker-local/check-deploy index 6c204c709..310b5c9a3 100755 --- a/plugins/scheduler-docker-local/check-deploy +++ b/plugins/scheduler-docker-local/check-deploy @@ -42,7 +42,7 @@ source "$PLUGIN_AVAILABLE_PATH/config/functions" trigger-scheduler-docker-local-check-deploy() { declare desc="scheduler-docker-local check-deploy plugin trigger" declare trigger="check-deploy" - declare APP="$1" DOKKU_APP_CONTAINER_ID="$2" DOKKU_APP_CONTAINER_TYPE="$3" DOKKU_APP_LISTEN_PORT="$4" DOKKU_APP_LISTEN_IP="$5" + declare APP="$1" DOKKU_APP_CONTAINER_ID="$2" DOKKU_APP_CONTAINER_TYPE="$3" DOKKU_APP_LISTEN_PORT="$4" DOKKU_APP_LISTEN_IP="$5" CONTAINER_INDEX="$6" local DOKKU_SCHEDULER=$(get_app_scheduler "$APP") if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then @@ -65,7 +65,7 @@ trigger-scheduler-docker-local-check-deploy() { eval "$(config_export app "$APP")" if [[ "$(is_app_proctype_checks_skipped "$APP" "$DOKKU_APP_CONTAINER_TYPE")" == "true" ]]; then - dokku_log_info2_quiet "Zero downtime checks for app ($APP) have been skipped. moving on..." + dokku_log_info2_quiet "Zero downtime checks have been skipped ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" exit 0 fi @@ -87,9 +87,9 @@ trigger-scheduler-docker-local-check-deploy() { local id="$1" rm -rf "$CHECK_DEPLOY_TMP_WORK_DIR" &>/dev/null || true if [[ $id ]]; then - dokku_log_info2_quiet "Start of $APP $DOKKU_APP_CONTAINER_TYPE container output:" + dokku_log_info2_quiet "Start of $APP container output ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" dokku_container_log_verbose_quiet "$id" - dokku_log_info2_quiet "End of $APP $DOKKU_APP_CONTAINER_TYPE container output" + dokku_log_info2_quiet "End of $APP container output ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" fi } trap "checks_check_deploy_cleanup $DOKKU_APP_CONTAINER_ID" RETURN INT TERM EXIT @@ -100,19 +100,19 @@ trigger-scheduler-docker-local-check-deploy() { # simple default check to see if the container stuck around local DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}" - dokku_log_verbose "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ..." + dokku_log_verbose "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" sleep "$DOKKU_DEFAULT_CHECKS_WAIT" - ! (is_container_status "$DOKKU_APP_CONTAINER_ID" "Running") && dokku_log_fail "App container failed to start!!" + ! (is_container_status "$DOKKU_APP_CONTAINER_ID" "Running") && dokku_log_fail "App container failed to start ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" local container_restarts="$("$DOCKER_BIN" container inspect --format "{{ .RestartCount }}" "$DOKKU_APP_CONTAINER_ID")" if [[ $container_restarts -ne 0 ]]; then "$DOCKER_BIN" container update --restart=no "$DOKKU_APP_CONTAINER_ID" &>/dev/null || true "$DOCKER_BIN" container stop "$DOKKU_APP_CONTAINER_ID" || true - dokku_log_fail "App container failed to start!!" + dokku_log_fail "App container failed to start ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" fi trap - EXIT - dokku_log_verbose "Default container check successful!" && exit 0 + dokku_log_verbose "Default container check successful ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" && exit 0 fi local NEW_CHECKS_FILENAME="${CHECK_DEPLOY_TMP_WORK_DIR}/NEW_CHECKS" @@ -194,15 +194,13 @@ trigger-scheduler-docker-local-check-deploy() { # CURL_ARGS includes the behinds the scenes options for hitting the container service directly local CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS" - dokku_log_verbose "CHECKS expected result:" - dokku_log_verbose "$LOG_URL => \"$EXPECTED\"" - [[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS" + dokku_log_verbose "CHECKS expected result: $LOG_URL => \"$EXPECTED\" ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" local ATTEMPT=0 local SUCCESS=0 until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]; do local ATTEMPT=$((ATTEMPT + 1)) - dokku_log_verbose "Attempt $ATTEMPT/$ATTEMPTS. Waiting for $WAIT seconds ..." + dokku_log_verbose "Attempt $ATTEMPT/$ATTEMPTS. Waiting for $WAIT seconds ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" sleep "$WAIT" # Capture HTTP response or CURL error message @@ -214,13 +212,13 @@ trigger-scheduler-docker-local-check-deploy() { SUCCESS=1 break fi - dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\"" + dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\" ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" else # Failed to connect/no response, OUTPUT contains error message dokku_log_warn "$OUTPUT" fi - dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed." + dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" done if [[ $SUCCESS -ne 1 ]]; then @@ -229,12 +227,12 @@ trigger-scheduler-docker-local-check-deploy() { done if [[ $FAILEDCHECKS -gt 0 ]]; then - dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks." + dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" exit 1 fi trap - EXIT - dokku_log_verbose "All checks successful!" + dokku_log_verbose "All checks successful ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" } trigger-scheduler-docker-local-check-deploy "$@" diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index bb404bf90..041d0935b 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -21,7 +21,7 @@ trigger-scheduler-docker-local-scheduler-deploy() { local DOKKU_DOCKER_STOP_TIMEOUT DOKKU_HEROKUISH DOKKU_NETWORK_BIND_ALL IMAGE DOKKU_HEROKUISH=false DOKKU_CNB=false - IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") + IMAGE="$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")" is_image_cnb_based "$IMAGE" && DOKKU_CNB=true is_image_herokuish_based "$IMAGE" "$APP" && DOKKU_HEROKUISH=true @@ -51,132 +51,15 @@ trigger-scheduler-docker-local-scheduler-deploy() { while read -r line || [[ -n "$line" ]]; do PROC_TYPE=${line%%=*} PROC_COUNT=${line#*=} - CONTAINER_INDEX=1 - if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "true" ]]; then - dokku_log_info1 "Zero downtime is disabled for app ($APP.$PROC_TYPE), Stopping currently running containers" - local cid proctype_oldids="$(get_app_running_container_ids "$APP" "$PROC_TYPE" 2>/dev/null)" - for cid in $proctype_oldids; do - dokku_log_verbose "Stopping $APP.$PROC_TYPE ($cid)" - - # Retire the containers to ensure they get removed - plugn trigger scheduler-register-retired "$APP" "$cid" "$WAIT" - - # Disable the container restart policy - "$DOCKER_BIN" container update --restart=no "$cid" &>/dev/null || true - - # shellcheck disable=SC2086 - "$DOCKER_BIN" container stop $DOCKER_STOP_TIME_ARG "$cid" &>/dev/null - # remove cid from oldids to skip the old container finish processing - oldids="$(remove_val_from_list "$cid" "$oldids" " ")" - done - fi - - while [[ $CONTAINER_INDEX -le $PROC_COUNT ]]; do - local cid="" - local port="" - local ipaddr="" - local DOKKU_CONTAINER_ID_FILE="$DOKKU_ROOT/$APP/CONTAINER.$PROC_TYPE.$CONTAINER_INDEX" - local DYNO="$PROC_TYPE.$CONTAINER_INDEX" - - # start the app - local DOCKER_ARGS - DOCKER_ARGS=$(: | plugn trigger docker-args-deploy "$APP" "$IMAGE_TAG" "$PROC_TYPE" "$CONTAINER_INDEX") - DOCKER_ARGS+=" --label=com.dokku.process-type=$PROC_TYPE --label=com.dokku.dyno=$DYNO" - DOCKER_ARGS+=" --env=DYNO=$DYNO" - DOCKER_ARGS+=" --name=$APP.$DYNO.upcoming-$RANDOM" - DOCKER_ARGS+=" --init" - DOCKER_ARGS+=" $DOCKER_RUN_LABEL_ARGS $DOKKU_GLOBAL_RUN_ARGS " - DOCKER_ARGS+=$(: | plugn trigger docker-args-process-deploy "$APP" "$IMAGE_SOURCE_TYPE" "$IMAGE_TAG" "$PROC_TYPE" "$CONTAINER_INDEX") - [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" --env=TRACE=true" - - local START_CMD - [[ "$DOKKU_HEROKUISH" == "true" ]] && START_CMD="/start $PROC_TYPE" - [[ "$DOKKU_CNB" == "true" ]] && START_CMD="" - [[ -n "$DOKKU_START_CMD" ]] && START_CMD="$DOKKU_START_CMD" - - local DOKKU_PORT="" - if [[ "$PROC_TYPE" == "web" ]]; then - ports=($(plugn trigger network-compute-ports "$APP" "$PROC_TYPE" "$DOKKU_HEROKUISH" "$CONTAINER_INDEX")) - local DOKKU_DOCKER_PORT_ARGS="" - for p in "${ports[@]}"; do - if [[ ! "$p" =~ .*udp.* ]]; then - DOKKU_PORT=${DOKKU_PORT:="$p"} - fi - - if [[ "$DOKKU_NETWORK_BIND_ALL" == "true" ]]; then - DOCKER_ARGS+=" -p $p" - fi - done - fi - - DOCKER_ARGS+=" --env=PORT=$DOKKU_PORT" - - START_CMD=$(fn-scheduler-docker-local-extract-start-cmd "$APP" "$PROC_TYPE" "$START_CMD" "$DOKKU_HEROKUISH" "$DOKKU_PORT") - DOCKER_ARGS+=" $IMAGE" - DOCKER_ARGS+=" $START_CMD" - - declare -a ARG_ARRAY - eval "ARG_ARRAY=($DOCKER_ARGS)" - cid=$(fn-scheduler-docker-local-start-app-container "$APP" "${ARG_ARRAY[@]}") - - plugn trigger post-container-create "app" "$cid" "$APP" "deploy" "$PROC_TYPE" - "$DOCKER_BIN" container start "$cid" >/dev/null || true - - ipaddr=$(plugn trigger network-get-ipaddr "$APP" "$PROC_TYPE" "$cid") - port=$(plugn trigger network-get-port "$APP" "$PROC_TYPE" "$cid" "$DOKKU_HEROKUISH") - - kill_new() { - declare desc="wrapper function to kill newly started app container" - declare CID="$1" PROC_TYPE="$2" CONTAINER_INDEX="$3" - - plugn trigger scheduler-register-retired "$APP" "$CID" "$WAIT" - mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP" - echo "${CID} ${PROC_TYPE}.${CONTAINER_INDEX}" >>"${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers" - "$DOCKER_BIN" container inspect "$CID" &>/dev/null && { - # Disable the container restart policy - "$DOCKER_BIN" container update --restart=no "$CID" &>/dev/null || true - "$DOCKER_BIN" container stop "$CID" >/dev/null && "$DOCKER_BIN" container kill "$CID" &>/dev/null - } - trap - INT TERM EXIT - kill -9 $$ - } - - # run checks first, then post-deploy hooks, which switches proxy traffic - trap "kill_new $cid $PROC_TYPE $CONTAINER_INDEX" INT TERM EXIT - if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "false" ]]; then - dokku_log_info1 "Attempting pre-flight checks ($PROC_TYPE.$CONTAINER_INDEX)" - plugn trigger check-deploy "$APP" "$cid" "$PROC_TYPE" "$port" "$ipaddr" - fi - trap - INT TERM EXIT - - if [[ -f "$DOKKU_CONTAINER_ID_FILE" ]]; then - # schedule old container for retirement - dokku_log_verbose "Scheduling old container shutdown for $PROC_TYPE.$CONTAINER_INDEX in $WAIT seconds" - plugn trigger scheduler-register-retired "$APP" "$(cat "$DOKKU_CONTAINER_ID_FILE")" "$WAIT" - fi - - # now using the new container - [[ -n "$cid" ]] && echo "$cid" >"$DOKKU_CONTAINER_ID_FILE" - [[ -n "$ipaddr" ]] && plugn trigger network-write-ipaddr "$APP" "$PROC_TYPE" "$CONTAINER_INDEX" "$ipaddr" - [[ -n "$port" ]] && plugn trigger network-write-port "$APP" "$PROC_TYPE" "$CONTAINER_INDEX" "$port" - - # cleanup pre-migration files - rm -f "$DOKKU_ROOT/$APP/CONTAINER" "$DOKKU_ROOT/$APP/IP" "$DOKKU_ROOT/$APP/PORT" - - local CONTAINER_INDEX=$((CONTAINER_INDEX + 1)) - done - # cleanup when we scale down - if [[ "$PROC_COUNT" == 0 ]]; then - local CONTAINER_IDX_OFFSET=0 - else - local CONTAINER_IDX_OFFSET=$((PROC_COUNT + 1)) - fi - local container_state_filetype - for container_state_filetype in CONTAINER IP PORT; do - cd "$DOKKU_ROOT/$APP" - find . -maxdepth 1 -name "$container_state_filetype.$PROC_TYPE.*" -printf "%f\n" | sort -t . -k 3 -n | tail -n +$CONTAINER_IDX_OFFSET | xargs rm -f - done + dokku_log_info1 "Deploying $PROC_TYPE (count=$PROC_COUNT)" + export DOKKU_NETWORK_BIND_ALL="$DOKKU_NETWORK_BIND_ALL" + export DOKKU_HEROKUISH="$DOKKU_HEROKUISH" + export DOKKU_CNB="$DOKKU_CNB" + export DOCKER_RUN_LABEL_ARGS="$DOCKER_RUN_LABEL_ARGS" + export DOKKU_START_CMD="$DOKKU_START_CMD" + export DOCKER_STOP_TIME_ARG="$DOCKER_STOP_TIME_ARG" + "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/bin/scheduler-deploy-process" "$APP" "$IMAGE_SOURCE_TYPE" "$IMAGE" "$IMAGE_TAG" "$PROC_TYPE" "$PROC_COUNT" done < <(plugn trigger ps-current-scale "$APP") dokku_log_info1 "Running post-deploy" @@ -192,6 +75,9 @@ trigger-scheduler-docker-local-scheduler-deploy() { trap '' INT HUP sleep "$WAIT" for oldid in $oldids; do + if ! docker container inspect "$oldid" >/dev/null 2>&1; then + continue + fi # Disable the container restart policy "$DOCKER_BIN" container update --restart=no "$oldid" &>/dev/null || true @@ -211,18 +97,4 @@ trigger-scheduler-docker-local-scheduler-deploy() { fi } -fn-scheduler-docker-local-extract-start-cmd() { - declare APP="$1" PROC_TYPE="$2" START_CMD="$3" DOKKU_HEROKUISH="$4" PORT="$5" - local DOKKU_DOCKERFILE_START_CMD DOKKU_PROCFILE_START_CMD START_CMD - if [[ "$DOKKU_HEROKUISH" != "false" ]] && [[ -n "$START_CMD" ]]; then - echo "$START_CMD" - return - fi - - DOKKU_DOCKERFILE_START_CMD=$(config_get "$APP" DOKKU_DOCKERFILE_START_CMD || true) - DOKKU_PROCFILE_START_CMD=$(plugn trigger procfile-get-command "$APP" "$PROC_TYPE" "$PORT" 2>/dev/null || echo '') - START_CMD=${DOKKU_DOCKERFILE_START_CMD:-$DOKKU_PROCFILE_START_CMD} - echo "$START_CMD" -} - trigger-scheduler-docker-local-scheduler-deploy "$@"