Merge pull request #3784 from dokku/better-checks-context

Ensure checks attempts are tracked per-check instead of globally
2019-11-27 15:19:12 -05:00
parent 9078dabc82 ff4b0de04c
commit 2429503b21
3 changed files with 82 additions and 68 deletions
--- a/plugins/scheduler-docker-local/check-deploy
+++ b/plugins/scheduler-docker-local/check-deploy
@@ -79,7 +79,6 @@ scheduler-docker-local-check-deploy() {
  local TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
  # use this number of retries for checks
  local ATTEMPTS="${DOKKU_CHECKS_ATTEMPTS:-5}"
-  local CHECKS_URL="${DOKKU_CHECKS_URL:-http://dokku.viewdocs.io/dokku/deployment/zero-downtime-deploys/}"

  local CHECK_DEPLOY_TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
  local CHECKS_FILENAME=${CHECK_DEPLOY_TMP_WORK_DIR}/CHECKS
@@ -99,20 +98,13 @@ scheduler-docker-local-check-deploy() {
  }
  trap "checks_check_deploy_cleanup $DOKKU_APP_CONTAINER_ID" RETURN INT TERM EXIT

+  # We allow custom check for web instances only
  if [[ ! -s "${CHECKS_FILENAME}" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]]; then
-    # We allow custom check for web instances only
-    if [[ "$DOKKU_APP_CONTAINER_TYPE" == "web" ]]; then
-      dokku_log_verbose "CHECKS file not found in container: Running simple container check..."
-      dokku_log_verbose "For more efficient zero downtime deployments, create a file CHECKS. See ${CHECKS_URL} for examples"
-    else
-      dokku_log_verbose "Non web container detected: Running default container check..."
-    fi
-
    rm -rf "$CHECK_DEPLOY_TMP_WORK_DIR" &>/dev/null || true

    # simple default check to see if the container stuck around
    local DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}"
-    dokku_log_info1 "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ..."
+    dokku_log_verbose "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ..."
    sleep "$DOKKU_DEFAULT_CHECKS_WAIT"

    ! (is_container_status "$DOKKU_APP_CONTAINER_ID" "Running") && dokku_log_fail "App container failed to start!!"
@@ -124,7 +116,7 @@ scheduler-docker-local-check-deploy() {
    fi

    trap - EXIT
-    dokku_log_info1 "Default container check successful!" && exit 0
+    dokku_log_verbose "Default container check successful!" && exit 0
  fi

  # ensure CHECKS file has trailing newline
@@ -155,13 +147,40 @@ scheduler-docker-local-check-deploy() {
    fi
  done

-  local ATTEMPT=0
+  exec <"$CHECKS_FILENAME"
+  local CHECK_URL
+  local EXPECTED
+  local FAILEDCHECKS=0
+  while read -r CHECK_URL EXPECTED; do
+    # Ignore empty lines and lines starting with #
+    # shellcheck disable=SC1001
+    [[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue
+    # Ignore if it's not a URL in a supported format
+    # shellcheck disable=SC1001
+    ! [[ "$CHECK_URL" =~ ^(http(s)?:)?\/.* ]] && continue

-  until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]; do
-    local FAILEDCHECKS=0
-    local ATTEMPT=$((ATTEMPT + 1))
-    dokku_log_info1 "Attempt $ATTEMPT/$ATTEMPTS Waiting for $WAIT seconds ..."
-    sleep "$WAIT"
+    if [[ "$CHECK_URL" =~ ^https?: ]]; then
+      local URL_PROTOCOL=${CHECK_URL%:*}
+      local CHECK_URL=${CHECK_URL#*:}
+    else
+      local URL_PROTOCOL="http"
+    fi
+
+    if [[ "$CHECK_URL" =~ ^//.+ ]]; then
+      # To test a URL with specific host name, we still make request to localhost,
+      # but we set Host header to $SEND_HOST.
+      #
+      # The pattern is
+      #   //SEND_HOST/PATHNAME
+      local UNPREFIXED=${CHECK_URL#//}
+      local URL_HOSTNAME=${UNPREFIXED%%/*}
+      local URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME}
+
+      local HEADERS="-H Host:$URL_HOSTNAME"
+    else
+      local URL_HOSTNAME=localhost
+      local URL_PATHNAME=$CHECK_URL
+    fi

    # -q           Do not use .curlrc (must come first)
    # --compressed Test compression handled correctly
@@ -176,48 +195,20 @@ scheduler-docker-local-check-deploy() {
      local CURL_OPTIONS+=" -H X-Forwarded-Proto:https"
    fi

-    exec <"$CHECKS_FILENAME"
-    local CHECK_URL
-    local EXPECTED
-    while read -r CHECK_URL EXPECTED; do
-      # Ignore empty lines and lines starting with #
-      # shellcheck disable=SC1001
-      [[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue
-      # Ignore if it's not a URL in a supported format
-      # shellcheck disable=SC1001
-      ! [[ "$CHECK_URL" =~ ^(http(s)?:)?\/.* ]] && continue
+    # LOG_URL is used in verbose output below
+    local LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME"
+    # CURL_ARGS includes the behinds the scenes options for hitting the container service directly
+    local CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"

-      if [[ "$CHECK_URL" =~ ^https?: ]]; then
-        local URL_PROTOCOL=${CHECK_URL%:*}
-        local CHECK_URL=${CHECK_URL#*:}
-      else
-        local URL_PROTOCOL="http"
-      fi
+    dokku_log_verbose "CHECKS expected result:"
+    dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
+    [[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"

-      if [[ "$CHECK_URL" =~ ^//.+ ]]; then
-        # To test a URL with specific host name, we still make request to localhost,
-        # but we set Host header to $SEND_HOST.
-        #
-        # The pattern is
-        #   //SEND_HOST/PATHNAME
-        local UNPREFIXED=${CHECK_URL#//}
-        local URL_HOSTNAME=${UNPREFIXED%%/*}
-        local URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME}
-
-        local HEADERS="-H Host:$URL_HOSTNAME"
-      else
-        local URL_HOSTNAME=localhost
-        local URL_PATHNAME=$CHECK_URL
-      fi
-
-      # This URL will show up in the messages
-      local LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME"
-      # And how we formulate the CURL request
-      local CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"
-
-      dokku_log_verbose "CHECKS expected result:"
-      dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
-      [[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"
+    local ATTEMPT=0
+    until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]; do
+      local ATTEMPT=$((ATTEMPT + 1))
+      dokku_log_verbose "Attempt $ATTEMPT/$ATTEMPTS. Waiting for $WAIT seconds ..."
+      sleep "$WAIT"

      # Capture HTTP response or CURL error message
      # shellcheck disable=SC2086
@@ -232,14 +223,14 @@ scheduler-docker-local-check-deploy() {
        dokku_log_warn "$OUTPUT"
        local FAILEDCHECKS=$((FAILEDCHECKS + 1))
      fi
-    done

-    if [[ $FAILEDCHECKS -gt 0 ]]; then
-      dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed."
-      local SUCCESS=0
-    else
-      local SUCCESS=1
-    fi
+      if [[ $FAILEDCHECKS -gt 0 ]]; then
+        dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed."
+        local SUCCESS=0
+      else
+        local SUCCESS=1
+      fi
+    done
  done

  if [[ $FAILEDCHECKS -gt 0 ]]; then
@@ -248,7 +239,7 @@ scheduler-docker-local-check-deploy() {
  fi

  trap - EXIT
-  dokku_log_info1 "All checks successful!"
+  dokku_log_verbose "All checks successful!"
 }

 scheduler-docker-local-check-deploy "$@"
--- a/plugins/scheduler-docker-local/pre-deploy
+++ b/plugins/scheduler-docker-local/pre-deploy
@@ -9,15 +9,22 @@ scheduler-docker-local-pre-deploy() {
  declare desc="scheduler-docker-local pre-deploy plugin trigger"
  declare trigger="scheduler-docker-local pre-deploy"
  declare APP="$1" IMAGE_TAG="$2"
-  local IMAGE DISABLE_CHOWN DOCKER_ARGS DOKKU_APP_TYPE DOKKU_APP_USER APP_PATHS CONTAINER_PATHS
-  declare -a ARG_ARRAY

  local DOKKU_SCHEDULER=$(get_app_scheduler "$APP")
  if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then
    return
  fi

+  scheduler-docker-local-pre-deploy-chown-app "$APP" "$IMAGE_TAG"
+  scheduler-docker-local-pre-deploy-precheck "$APP" "$IMAGE_TAG"
+}
+
+scheduler-docker-local-pre-deploy-chown-app() {
+  declare desc="Runs chown against the /app directory for herokuish images"
+  declare APP="$1" IMAGE_TAG="$2"
  local DOCKER_RUN_LABEL_ARGS="--label=com.dokku.app-name=$APP"
+  local IMAGE DISABLE_CHOWN DOCKER_ARGS DOKKU_APP_TYPE DOKKU_APP_USER APP_PATHS CONTAINER_PATHS
+  declare -a ARG_ARRAY

  IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG")

@@ -47,4 +54,20 @@ scheduler-docker-local-pre-deploy() {
  "$DOCKER_BIN" container run "${DOCKER_RUN_LABEL_ARGS[@]}" $DOKKU_GLOBAL_RUN_ARGS "${ARG_ARRAY[@]}" $IMAGE /bin/bash -c "find $CONTAINER_PATHS -not -user $DOKKU_APP_USER -print0 | xargs -0 -r chown -R $DOKKU_APP_USER" || true
 }

+scheduler-docker-local-pre-deploy-precheck() {
+  declare desc="Outputs the checks messages if necessary"
+  declare APP="$1" IMAGE_TAG="$2"
+  local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")
+  local CHECKS_FILE=$(mktemp "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
+  trap "rm -rf '$CHECKS_FILE' >/dev/null" RETURN INT TERM
+  copy_from_image "$IMAGE" "CHECKS" "$CHECKS_FILENAME" 2>/dev/null || true
+
+  dokku_log_info2 "Processing deployment checks"
+  if [[ ! -s "${CHECKS_FILENAME}" ]]; then
+    local CHECKS_URL="${DOKKU_CHECKS_URL:-http://dokku.viewdocs.io/dokku/deployment/zero-downtime-deploys/}"
+    dokku_log_verbose "No CHECKS file found. Simple container checks will be performed."
+    dokku_log_verbose "For more efficient zero downtime deployments, create a CHECKS file. See ${CHECKS_URL} for examples"
+  fi
+}
+
 scheduler-docker-local-pre-deploy "$@"
--- a/plugins/scheduler-docker-local/scheduler-deploy
+++ b/plugins/scheduler-docker-local/scheduler-deploy
@@ -130,7 +130,7 @@ scheduler-docker-local-scheduler-deploy() {
      # run checks first, then post-deploy hooks, which switches proxy traffic
      trap 'kill_new $cid $PROC_TYPE $CONTAINER_INDEX' INT TERM EXIT
      if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "false" ]]; then
-        dokku_log_info1 "Attempting pre-flight checks"
+        dokku_log_info1 "Attempting pre-flight checks ($PROC_TYPE.$CONTAINER_INDEX)"
        plugn trigger check-deploy "$APP" "$cid" "$PROC_TYPE" "$port" "$ipaddr"
      fi
      trap - INT TERM EXIT