refactor: split out docker-local scheduler deploys into separate binaries

This will allow those binaries to run in parallel.

plugins/scheduler-docker-local/check-deploy

@@ -42,7 +42,7 @@ source "$PLUGIN_AVAILABLE_PATH/config/functions"
 trigger-scheduler-docker-local-check-deploy() {
   declare desc="scheduler-docker-local check-deploy plugin trigger"
   declare trigger="check-deploy"
-  declare APP="$1" DOKKU_APP_CONTAINER_ID="$2" DOKKU_APP_CONTAINER_TYPE="$3" DOKKU_APP_LISTEN_PORT="$4" DOKKU_APP_LISTEN_IP="$5"
+  declare APP="$1" DOKKU_APP_CONTAINER_ID="$2" DOKKU_APP_CONTAINER_TYPE="$3" DOKKU_APP_LISTEN_PORT="$4" DOKKU_APP_LISTEN_IP="$5" CONTAINER_INDEX="$6"
 
   local DOKKU_SCHEDULER=$(get_app_scheduler "$APP")
   if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then
@@ -65,7 +65,7 @@ trigger-scheduler-docker-local-check-deploy() {
   eval "$(config_export app "$APP")"
 
   if [[ "$(is_app_proctype_checks_skipped "$APP" "$DOKKU_APP_CONTAINER_TYPE")" == "true" ]]; then
-    dokku_log_info2_quiet "Zero downtime checks for app ($APP) have been skipped. moving on..."
+    dokku_log_info2_quiet "Zero downtime checks have been skipped ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
     exit 0
   fi
 
@@ -87,9 +87,9 @@ trigger-scheduler-docker-local-check-deploy() {
     local id="$1"
     rm -rf "$CHECK_DEPLOY_TMP_WORK_DIR" &>/dev/null || true
     if [[ $id ]]; then
-      dokku_log_info2_quiet "Start of $APP $DOKKU_APP_CONTAINER_TYPE container output:"
+      dokku_log_info2_quiet "Start of $APP container output ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
       dokku_container_log_verbose_quiet "$id"
-      dokku_log_info2_quiet "End of $APP $DOKKU_APP_CONTAINER_TYPE container output"
+      dokku_log_info2_quiet "End of $APP container output ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
     fi
   }
   trap "checks_check_deploy_cleanup $DOKKU_APP_CONTAINER_ID" RETURN INT TERM EXIT
@@ -100,19 +100,19 @@ trigger-scheduler-docker-local-check-deploy() {
 
     # simple default check to see if the container stuck around
     local DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}"
-    dokku_log_verbose "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ..."
+    dokku_log_verbose "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
     sleep "$DOKKU_DEFAULT_CHECKS_WAIT"
 
-    ! (is_container_status "$DOKKU_APP_CONTAINER_ID" "Running") && dokku_log_fail "App container failed to start!!"
+    ! (is_container_status "$DOKKU_APP_CONTAINER_ID" "Running") && dokku_log_fail "App container failed to start ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
     local container_restarts="$("$DOCKER_BIN" container inspect --format "{{ .RestartCount }}" "$DOKKU_APP_CONTAINER_ID")"
     if [[ $container_restarts -ne 0 ]]; then
       "$DOCKER_BIN" container update --restart=no "$DOKKU_APP_CONTAINER_ID" &>/dev/null || true
       "$DOCKER_BIN" container stop "$DOKKU_APP_CONTAINER_ID" || true
-      dokku_log_fail "App container failed to start!!"
+      dokku_log_fail "App container failed to start ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
     fi
 
     trap - EXIT
-    dokku_log_verbose "Default container check successful!" && exit 0
+    dokku_log_verbose "Default container check successful ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)" && exit 0
   fi
 
   local NEW_CHECKS_FILENAME="${CHECK_DEPLOY_TMP_WORK_DIR}/NEW_CHECKS"
@@ -194,15 +194,13 @@ trigger-scheduler-docker-local-check-deploy() {
     # CURL_ARGS includes the behinds the scenes options for hitting the container service directly
     local CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"
 
-    dokku_log_verbose "CHECKS expected result:"
+    dokku_log_verbose "CHECKS expected result: $LOG_URL => \"$EXPECTED\" ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
-    dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
-    [[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"
 
     local ATTEMPT=0
     local SUCCESS=0
     until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]; do
       local ATTEMPT=$((ATTEMPT + 1))
-      dokku_log_verbose "Attempt $ATTEMPT/$ATTEMPTS. Waiting for $WAIT seconds ..."
+      dokku_log_verbose "Attempt $ATTEMPT/$ATTEMPTS. Waiting for $WAIT seconds ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
       sleep "$WAIT"
 
       # Capture HTTP response or CURL error message
@@ -214,13 +212,13 @@ trigger-scheduler-docker-local-check-deploy() {
           SUCCESS=1
           break
         fi
-        dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
+        dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\" ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
       else
         # Failed to connect/no response, OUTPUT contains error message
         dokku_log_warn "$OUTPUT"
       fi
 
-      dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed."
+      dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
     done
 
     if [[ $SUCCESS -ne 1 ]]; then
@@ -229,12 +227,12 @@ trigger-scheduler-docker-local-check-deploy() {
   done
 
   if [[ $FAILEDCHECKS -gt 0 ]]; then
-    dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks."
+    dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
     exit 1
   fi
 
   trap - EXIT
-  dokku_log_verbose "All checks successful!"
+  dokku_log_verbose "All checks successful ($DOKKU_APP_CONTAINER_TYPE.$CONTAINER_INDEX)"
 }
 
 trigger-scheduler-docker-local-check-deploy "$@"

plugins/scheduler-docker-local/scheduler-deploy

@@ -21,7 +21,7 @@ trigger-scheduler-docker-local-scheduler-deploy() {
   local DOKKU_DOCKER_STOP_TIMEOUT DOKKU_HEROKUISH DOKKU_NETWORK_BIND_ALL IMAGE
   DOKKU_HEROKUISH=false
   DOKKU_CNB=false
-  IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")
+  IMAGE="$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")"
 
   is_image_cnb_based "$IMAGE" && DOKKU_CNB=true
   is_image_herokuish_based "$IMAGE" "$APP" && DOKKU_HEROKUISH=true
@@ -51,132 +51,15 @@ trigger-scheduler-docker-local-scheduler-deploy() {
   while read -r line || [[ -n "$line" ]]; do
     PROC_TYPE=${line%%=*}
     PROC_COUNT=${line#*=}
-    CONTAINER_INDEX=1
 
-    if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "true" ]]; then
+    dokku_log_info1 "Deploying $PROC_TYPE (count=$PROC_COUNT)"
-      dokku_log_info1 "Zero downtime is disabled for app ($APP.$PROC_TYPE), Stopping currently running containers"
+    export DOKKU_NETWORK_BIND_ALL="$DOKKU_NETWORK_BIND_ALL"
-      local cid proctype_oldids="$(get_app_running_container_ids "$APP" "$PROC_TYPE" 2>/dev/null)"
+    export DOKKU_HEROKUISH="$DOKKU_HEROKUISH"
-      for cid in $proctype_oldids; do
+    export DOKKU_CNB="$DOKKU_CNB"
-        dokku_log_verbose "Stopping $APP.$PROC_TYPE ($cid)"
+    export DOCKER_RUN_LABEL_ARGS="$DOCKER_RUN_LABEL_ARGS"
-
+    export DOKKU_START_CMD="$DOKKU_START_CMD"
-        # Retire the containers to ensure they get removed
+    export DOCKER_STOP_TIME_ARG="$DOCKER_STOP_TIME_ARG"
-        plugn trigger scheduler-register-retired "$APP" "$cid" "$WAIT"
+    "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/bin/scheduler-deploy-process" "$APP" "$IMAGE_SOURCE_TYPE" "$IMAGE" "$IMAGE_TAG" "$PROC_TYPE" "$PROC_COUNT"
-
-        # Disable the container restart policy
-        "$DOCKER_BIN" container update --restart=no "$cid" &>/dev/null || true
-
-        # shellcheck disable=SC2086
-        "$DOCKER_BIN" container stop $DOCKER_STOP_TIME_ARG "$cid" &>/dev/null
-        # remove cid from oldids to skip the old container finish processing
-        oldids="$(remove_val_from_list "$cid" "$oldids" " ")"
-      done
-    fi
-
-    while [[ $CONTAINER_INDEX -le $PROC_COUNT ]]; do
-      local cid=""
-      local port=""
-      local ipaddr=""
-      local DOKKU_CONTAINER_ID_FILE="$DOKKU_ROOT/$APP/CONTAINER.$PROC_TYPE.$CONTAINER_INDEX"
-      local DYNO="$PROC_TYPE.$CONTAINER_INDEX"
-
-      # start the app
-      local DOCKER_ARGS
-      DOCKER_ARGS=$(: | plugn trigger docker-args-deploy "$APP" "$IMAGE_TAG" "$PROC_TYPE" "$CONTAINER_INDEX")
-      DOCKER_ARGS+=" --label=com.dokku.process-type=$PROC_TYPE --label=com.dokku.dyno=$DYNO"
-      DOCKER_ARGS+=" --env=DYNO=$DYNO"
-      DOCKER_ARGS+=" --name=$APP.$DYNO.upcoming-$RANDOM"
-      DOCKER_ARGS+=" --init"
-      DOCKER_ARGS+=" $DOCKER_RUN_LABEL_ARGS $DOKKU_GLOBAL_RUN_ARGS "
-      DOCKER_ARGS+=$(: | plugn trigger docker-args-process-deploy "$APP" "$IMAGE_SOURCE_TYPE" "$IMAGE_TAG" "$PROC_TYPE" "$CONTAINER_INDEX")
-      [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" --env=TRACE=true"
-
-      local START_CMD
-      [[ "$DOKKU_HEROKUISH" == "true" ]] && START_CMD="/start $PROC_TYPE"
-      [[ "$DOKKU_CNB" == "true" ]] && START_CMD=""
-      [[ -n "$DOKKU_START_CMD" ]] && START_CMD="$DOKKU_START_CMD"
-
-      local DOKKU_PORT=""
-      if [[ "$PROC_TYPE" == "web" ]]; then
-        ports=($(plugn trigger network-compute-ports "$APP" "$PROC_TYPE" "$DOKKU_HEROKUISH" "$CONTAINER_INDEX"))
-        local DOKKU_DOCKER_PORT_ARGS=""
-        for p in "${ports[@]}"; do
-          if [[ ! "$p" =~ .*udp.* ]]; then
-            DOKKU_PORT=${DOKKU_PORT:="$p"}
-          fi
-
-          if [[ "$DOKKU_NETWORK_BIND_ALL" == "true" ]]; then
-            DOCKER_ARGS+=" -p $p"
-          fi
-        done
-      fi
-
-      DOCKER_ARGS+=" --env=PORT=$DOKKU_PORT"
-
-      START_CMD=$(fn-scheduler-docker-local-extract-start-cmd "$APP" "$PROC_TYPE" "$START_CMD" "$DOKKU_HEROKUISH" "$DOKKU_PORT")
-      DOCKER_ARGS+=" $IMAGE"
-      DOCKER_ARGS+=" $START_CMD"
-
-      declare -a ARG_ARRAY
-      eval "ARG_ARRAY=($DOCKER_ARGS)"
-      cid=$(fn-scheduler-docker-local-start-app-container "$APP" "${ARG_ARRAY[@]}")
-
-      plugn trigger post-container-create "app" "$cid" "$APP" "deploy" "$PROC_TYPE"
-      "$DOCKER_BIN" container start "$cid" >/dev/null || true
-
-      ipaddr=$(plugn trigger network-get-ipaddr "$APP" "$PROC_TYPE" "$cid")
-      port=$(plugn trigger network-get-port "$APP" "$PROC_TYPE" "$cid" "$DOKKU_HEROKUISH")
-
-      kill_new() {
-        declare desc="wrapper function to kill newly started app container"
-        declare CID="$1" PROC_TYPE="$2" CONTAINER_INDEX="$3"
-
-        plugn trigger scheduler-register-retired "$APP" "$CID" "$WAIT"
-        mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP"
-        echo "${CID} ${PROC_TYPE}.${CONTAINER_INDEX}" >>"${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers"
-        "$DOCKER_BIN" container inspect "$CID" &>/dev/null && {
-          # Disable the container restart policy
-          "$DOCKER_BIN" container update --restart=no "$CID" &>/dev/null || true
-          "$DOCKER_BIN" container stop "$CID" >/dev/null && "$DOCKER_BIN" container kill "$CID" &>/dev/null
-        }
-        trap - INT TERM EXIT
-        kill -9 $$
-      }
-
-      # run checks first, then post-deploy hooks, which switches proxy traffic
-      trap "kill_new $cid $PROC_TYPE $CONTAINER_INDEX" INT TERM EXIT
-      if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "false" ]]; then
-        dokku_log_info1 "Attempting pre-flight checks ($PROC_TYPE.$CONTAINER_INDEX)"
-        plugn trigger check-deploy "$APP" "$cid" "$PROC_TYPE" "$port" "$ipaddr"
-      fi
-      trap - INT TERM EXIT
-
-      if [[ -f "$DOKKU_CONTAINER_ID_FILE" ]]; then
-        # schedule old container for retirement
-        dokku_log_verbose "Scheduling old container shutdown for $PROC_TYPE.$CONTAINER_INDEX in $WAIT seconds"
-        plugn trigger scheduler-register-retired "$APP" "$(cat "$DOKKU_CONTAINER_ID_FILE")" "$WAIT"
-      fi
-
-      # now using the new container
-      [[ -n "$cid" ]] && echo "$cid" >"$DOKKU_CONTAINER_ID_FILE"
-      [[ -n "$ipaddr" ]] && plugn trigger network-write-ipaddr "$APP" "$PROC_TYPE" "$CONTAINER_INDEX" "$ipaddr"
-      [[ -n "$port" ]] && plugn trigger network-write-port "$APP" "$PROC_TYPE" "$CONTAINER_INDEX" "$port"
-
-      # cleanup pre-migration files
-      rm -f "$DOKKU_ROOT/$APP/CONTAINER" "$DOKKU_ROOT/$APP/IP" "$DOKKU_ROOT/$APP/PORT"
-
-      local CONTAINER_INDEX=$((CONTAINER_INDEX + 1))
-    done
-    # cleanup when we scale down
-    if [[ "$PROC_COUNT" == 0 ]]; then
-      local CONTAINER_IDX_OFFSET=0
-    else
-      local CONTAINER_IDX_OFFSET=$((PROC_COUNT + 1))
-    fi
-    local container_state_filetype
-    for container_state_filetype in CONTAINER IP PORT; do
-      cd "$DOKKU_ROOT/$APP"
-      find . -maxdepth 1 -name "$container_state_filetype.$PROC_TYPE.*" -printf "%f\n" | sort -t . -k 3 -n | tail -n +$CONTAINER_IDX_OFFSET | xargs rm -f
-    done
   done < <(plugn trigger ps-current-scale "$APP")
 
   dokku_log_info1 "Running post-deploy"
@@ -192,6 +75,9 @@ trigger-scheduler-docker-local-scheduler-deploy() {
       trap '' INT HUP
       sleep "$WAIT"
       for oldid in $oldids; do
+        if ! docker container inspect "$oldid" >/dev/null 2>&1; then
+          continue
+        fi
         # Disable the container restart policy
         "$DOCKER_BIN" container update --restart=no "$oldid" &>/dev/null || true
 
@@ -211,18 +97,4 @@ trigger-scheduler-docker-local-scheduler-deploy() {
   fi
 }
 
-fn-scheduler-docker-local-extract-start-cmd() {
-  declare APP="$1" PROC_TYPE="$2" START_CMD="$3" DOKKU_HEROKUISH="$4" PORT="$5"
-  local DOKKU_DOCKERFILE_START_CMD DOKKU_PROCFILE_START_CMD START_CMD
-  if [[ "$DOKKU_HEROKUISH" != "false" ]] && [[ -n "$START_CMD" ]]; then
-    echo "$START_CMD"
-    return
-  fi
-
-  DOKKU_DOCKERFILE_START_CMD=$(config_get "$APP" DOKKU_DOCKERFILE_START_CMD || true)
-  DOKKU_PROCFILE_START_CMD=$(plugn trigger procfile-get-command "$APP" "$PROC_TYPE" "$PORT" 2>/dev/null || echo '')
-  START_CMD=${DOKKU_DOCKERFILE_START_CMD:-$DOKKU_PROCFILE_START_CMD}
-  echo "$START_CMD"
-}
-
 trigger-scheduler-docker-local-scheduler-deploy "$@"