d1319977d6
This allows folks to later investigate why the rename failed (probably an app bug) and helpfully posts the container logs. If multiple containers fail, they may have a ton of output, but this is better than not knowing what is going on. Closes #3900
241 lines
8.7 KiB
Bash
Executable File
241 lines
8.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Hook to check server against list of checks specified in CHECKS file.
|
|
#
|
|
# The CHECKS file may contain empty lines, comments (lines starting with #),
|
|
# settings (NAME=VALUE) and check instructions.
|
|
#
|
|
# The format of a check instruction is a path, optionally followed by the
|
|
# expected content. For example:
|
|
#
|
|
# / My Amazing App
|
|
# /stylesheets/index.css .body
|
|
# /scripts/index.js $(function()
|
|
# /images/logo.png
|
|
#
|
|
# To check an application that supports multiple hostnames, use relative URLs
|
|
# that include the hostname, for example:
|
|
#
|
|
# //admin.example.com Admin Dashboard
|
|
# //static.example.com/logo.png
|
|
#
|
|
# You can also specify the protocol to explicitly check HTTPS requests.
|
|
#
|
|
# The default behavior is to wait for 5 seconds before running the first check,
|
|
# and timeout each check to 30 seconds.
|
|
#
|
|
# By default, checks will be retried 5 times.
|
|
|
|
# You can change these by setting WAIT, TIMEOUT and ATTEMPTS to different values, for
|
|
# example:
|
|
#
|
|
# WAIT=30 # Wait 1/2 minute
|
|
# TIMEOUT=60 # Timeout after a minute
|
|
# ATTEMPTS=10 # retry checks 10 times
|
|
#
|
|
set -eo pipefail
|
|
[[ $DOKKU_TRACE ]] && set -x
|
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
|
|
source "$PLUGIN_AVAILABLE_PATH/checks/functions"
|
|
source "$PLUGIN_AVAILABLE_PATH/config/functions"
|
|
|
|
trigger-scheduler-docker-local-check-deploy() {
|
|
declare desc="scheduler-docker-local check-deploy plugin trigger"
|
|
declare trigger="check-deploy"
|
|
declare APP="$1" DOKKU_APP_CONTAINER_ID="$2" DOKKU_APP_CONTAINER_TYPE="$3" DOKKU_APP_LISTEN_PORT="$4" DOKKU_APP_LISTEN_IP="$5"
|
|
|
|
local DOKKU_SCHEDULER=$(get_app_scheduler "$APP")
|
|
if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then
|
|
return
|
|
fi
|
|
|
|
if [[ -z "$DOKKU_APP_LISTEN_PORT" ]] && [[ -f "$DOKKU_ROOT/$APP/PORT" ]]; then
|
|
local DOKKU_APP_LISTEN_PORT=$(<"$DOKKU_ROOT/$APP/PORT")
|
|
fi
|
|
if [[ -z "$DOKKU_APP_LISTEN_IP" ]] && [[ -f "$DOKKU_ROOT/$APP/IP" ]]; then
|
|
local DOKKU_APP_LISTEN_IP=$(<"$DOKKU_ROOT/$APP/IP")
|
|
fi
|
|
if [[ -z "$DOKKU_APP_CONTAINER_ID" ]]; then
|
|
local DOKKU_APP_CIDS=($(get_app_container_ids "$APP"))
|
|
local DOKKU_APP_CONTAINER_ID=${DOKKU_APP_CIDS[0]}
|
|
fi
|
|
|
|
# source global and in-app envs to get DOKKU_CHECKS_WAIT and any other necessary vars
|
|
eval "$(config_export global)"
|
|
eval "$(config_export app "$APP")"
|
|
|
|
if [[ "$(is_app_proctype_checks_skipped "$APP" "$DOKKU_APP_CONTAINER_TYPE")" == "true" ]]; then
|
|
dokku_log_info2_quiet "Zero downtime checks for app ($APP) have been skipped. moving on..."
|
|
exit 0
|
|
fi
|
|
|
|
# Wait this many seconds (default 5) for server to start before running checks.
|
|
local WAIT="${DOKKU_CHECKS_WAIT:-5}"
|
|
# Wait this many seconds (default 30) for each response.
|
|
local TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
|
|
# use this number of retries for checks
|
|
local ATTEMPTS="${DOKKU_CHECKS_ATTEMPTS:-5}"
|
|
|
|
local CHECK_DEPLOY_TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
|
|
local CHECKS_FILENAME=${CHECK_DEPLOY_TMP_WORK_DIR}/CHECKS
|
|
local IMAGE_TAG="$(get_running_image_tag "$APP")"
|
|
local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")
|
|
copy_from_image "$IMAGE" "CHECKS" "$CHECKS_FILENAME" 2>/dev/null || true
|
|
|
|
checks_check_deploy_cleanup() {
|
|
declare desc="cleans up CHECK_DEPLOY_TMP_WORK_DIR and print container output"
|
|
local id="$1"
|
|
rm -rf "$CHECK_DEPLOY_TMP_WORK_DIR" &>/dev/null || true
|
|
if [[ $id ]]; then
|
|
dokku_log_info2_quiet "Start of $APP $DOKKU_APP_CONTAINER_TYPE container output:"
|
|
dokku_container_log_verbose_quiet "$id"
|
|
dokku_log_info2_quiet "End of $APP $DOKKU_APP_CONTAINER_TYPE container output"
|
|
fi
|
|
}
|
|
trap "checks_check_deploy_cleanup $DOKKU_APP_CONTAINER_ID" RETURN INT TERM EXIT
|
|
|
|
# We allow custom check for web instances only
|
|
if [[ ! -s "${CHECKS_FILENAME}" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]]; then
|
|
rm -rf "$CHECK_DEPLOY_TMP_WORK_DIR" &>/dev/null || true
|
|
|
|
# simple default check to see if the container stuck around
|
|
local DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}"
|
|
dokku_log_verbose "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ..."
|
|
sleep "$DOKKU_DEFAULT_CHECKS_WAIT"
|
|
|
|
! (is_container_status "$DOKKU_APP_CONTAINER_ID" "Running") && dokku_log_fail "App container failed to start!!"
|
|
local container_restarts="$("$DOCKER_BIN" container inspect --format "{{ .RestartCount }}" "$DOKKU_APP_CONTAINER_ID")"
|
|
if [[ $container_restarts -ne 0 ]]; then
|
|
"$DOCKER_BIN" container update --restart=no "$DOKKU_APP_CONTAINER_ID" &>/dev/null || true
|
|
"$DOCKER_BIN" container stop "$DOKKU_APP_CONTAINER_ID" || true
|
|
dokku_log_fail "App container failed to start!!"
|
|
fi
|
|
|
|
trap - EXIT
|
|
dokku_log_verbose "Default container check successful!" && exit 0
|
|
fi
|
|
|
|
local NEW_CHECKS_FILENAME="${CHECK_DEPLOY_TMP_WORK_DIR}/NEW_CHECKS"
|
|
template_checks() {
|
|
eval "$(config_export app "$APP" --merged)"
|
|
sigil -f "$CHECKS_FILENAME" | cat -s >"$NEW_CHECKS_FILENAME"
|
|
}
|
|
|
|
template_checks
|
|
CHECKS_FILENAME="$NEW_CHECKS_FILENAME"
|
|
|
|
# Reads name/value pairs, sets the WAIT and TIMEOUT variables
|
|
exec <"$CHECKS_FILENAME"
|
|
local line
|
|
local NAME
|
|
local VALUE
|
|
while read -r line; do
|
|
line=$(strip_inline_comments "$line")
|
|
# Name/value pair
|
|
if [[ "$line" =~ ^.+= ]]; then
|
|
NAME=${line%=*}
|
|
VALUE=${line#*=}
|
|
[[ "$NAME" == "WAIT" ]] && local WAIT=$VALUE
|
|
[[ "$NAME" == "TIMEOUT" ]] && local TIMEOUT=$VALUE
|
|
[[ "$NAME" == "ATTEMPTS" ]] && local ATTEMPTS=$VALUE
|
|
fi
|
|
done
|
|
|
|
exec <"$CHECKS_FILENAME"
|
|
local CHECK_URL
|
|
local EXPECTED
|
|
local FAILEDCHECKS=0
|
|
while read -r CHECK_URL EXPECTED; do
|
|
# Ignore empty lines and lines starting with #
|
|
# shellcheck disable=SC1001
|
|
[[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue
|
|
# Ignore if it's not a URL in a supported format
|
|
# shellcheck disable=SC1001
|
|
! [[ "$CHECK_URL" =~ ^(http(s)?:)?\/.* ]] && continue
|
|
|
|
if [[ "$CHECK_URL" =~ ^https?: ]]; then
|
|
local URL_PROTOCOL=${CHECK_URL%:*}
|
|
local CHECK_URL=${CHECK_URL#*:}
|
|
else
|
|
local URL_PROTOCOL="http"
|
|
fi
|
|
|
|
if [[ "$CHECK_URL" =~ ^//.+ ]]; then
|
|
# To test a URL with specific host name, we still make request to localhost,
|
|
# but we set Host header to $SEND_HOST.
|
|
#
|
|
# The pattern is
|
|
# //SEND_HOST/PATHNAME
|
|
local UNPREFIXED=${CHECK_URL#//}
|
|
local URL_HOSTNAME=${UNPREFIXED%%/*}
|
|
local URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME}
|
|
|
|
local HEADERS="-H Host:$URL_HOSTNAME"
|
|
else
|
|
local URL_HOSTNAME=localhost
|
|
local URL_PATHNAME=$CHECK_URL
|
|
fi
|
|
|
|
# -q Do not use .curlrc (must come first)
|
|
# --compressed Test compression handled correctly
|
|
# --fail Fail on server errors (4xx, 5xx)
|
|
# --location Follow redirects
|
|
# --noproxy Do not use http_proxy env variable
|
|
local CURL_OPTIONS="-q --compressed --fail --location --noproxy $DOKKU_APP_LISTEN_IP --max-time $TIMEOUT"
|
|
|
|
# Set X-Forwarded-Proto header if TLS is enabled.
|
|
local SSL="$DOKKU_ROOT/$APP/tls"
|
|
if [[ -e "$SSL/server.crt" && -e "$SSL/server.key" ]]; then
|
|
local CURL_OPTIONS+=" -H X-Forwarded-Proto:https"
|
|
fi
|
|
|
|
# LOG_URL is used in verbose output below
|
|
local LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME"
|
|
# CURL_ARGS includes the behinds the scenes options for hitting the container service directly
|
|
local CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"
|
|
|
|
dokku_log_verbose "CHECKS expected result:"
|
|
dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
|
|
[[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"
|
|
|
|
local ATTEMPT=0
|
|
local SUCCESS=0
|
|
until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]; do
|
|
local ATTEMPT=$((ATTEMPT + 1))
|
|
dokku_log_verbose "Attempt $ATTEMPT/$ATTEMPTS. Waiting for $WAIT seconds ..."
|
|
sleep "$WAIT"
|
|
|
|
# Capture HTTP response or CURL error message
|
|
# shellcheck disable=SC2086
|
|
if OUTPUT=$(curl -# $CURL_ARGS 2>&1); then
|
|
# OUTPUT contains the HTTP response
|
|
# shellcheck disable=SC2076
|
|
if [[ "$OUTPUT" =~ "$EXPECTED" ]]; then
|
|
SUCCESS=1
|
|
break
|
|
fi
|
|
dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
|
|
else
|
|
# Failed to connect/no response, OUTPUT contains error message
|
|
dokku_log_warn "$OUTPUT"
|
|
fi
|
|
|
|
dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed."
|
|
done
|
|
|
|
if [[ $SUCCESS -ne 1 ]]; then
|
|
FAILEDCHECKS=$((FAILEDCHECKS + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $FAILEDCHECKS -gt 0 ]]; then
|
|
dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks."
|
|
exit 1
|
|
fi
|
|
|
|
trap - EXIT
|
|
dokku_log_verbose "All checks successful!"
|
|
}
|
|
|
|
trigger-scheduler-docker-local-check-deploy "$@"
|