github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1056 from joshco/feature/retries

Browse Source 
Review: Feature/retries
This commit is contained in:
Jose Diaz-Gonzalez
2015-04-13 17:18:00 -04:00
parent 580bab491f b5eef489fd
commit 1a151ddabe
3 changed files with 293 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
plugins/checks/check-deploy
View File

@@ -24,11 +24,14 @@
# The default behavior is to wait for 5 seconds before running the first check,
# and timeout each check to 30 minutes.
#
# You can change these by setting WAIT and TIMEOUT to different values, for
# By default, checks will be retried 5 times.
# You can change these by setting WAIT, TIMEOUT and ATTEMPTS to different values, for
# example:
#
# WAIT=30 # Wait 1/2 minute
# TIMEOUT=60 # Timeout after a minute
# ATTEMPTS=10 # retry checks 10 times
#
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
@@ -54,6 +57,8 @@ fi
WAIT="${DOKKU_CHECKS_WAIT:-5}"
# Wait this many seconds (default 30) for each response.
TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
# use this number of retries for checks
ATTEMPTS="${DOKKU_CHECKS_ATTEMPTS:-5}"
# try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
# docker cp exits with status 1 when run as non-root user when it tries to chown the file
@@ -96,73 +101,92 @@ while read LINE ; do
VALUE=${TRIM#*=}
[[ "$NAME" = "WAIT" ]] && WAIT=$VALUE
[[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE
[[ "$NAME" = "ATTEMPTS" ]] && ATTEMPTS=$VALUE
fi
done
ATTEMPT=0
dokku_log_info1 "Waiting for $WAIT seconds ..."
sleep $WAIT
# -q do not use .curlrc (must come first)
# --compressed Test compression handled correctly
# --fail Fail on server errors (4xx, 5xx)
# --location Follow redirects
CURL_OPTIONS="-q --compressed --fail --location --max-time $TIMEOUT"
exec < "$FILENAME"
while read CHECK_URL EXPECTED ; do
# Ignore empty lines and lines starting with #
# shellcheck disable=SC1001
[[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue
# Ignore variables
[[ "$CHECK_URL" =~ ^.+= ]] && continue
if [[ "$CHECK_URL" =~ ^https?: ]] ; then
URL_PROTOCOL=${CHECK_URL%:*}
CHECK_URL=${CHECK_URL#*:}
else
URL_PROTOCOL="http"
fi
if [[ "$CHECK_URL" =~ ^//.+ ]] ; then
# To test a URL with specific host name, we still make request to localhost,
# but we set Host header to $SEND_HOST.
#
# The pattern is
# //SEND_HOST/PATHNAME
UNPREFIXED=${CHECK_URL#//}
URL_HOSTNAME=${UNPREFIXED%%/*}
URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME}
HEADERS="-H Host:$URL_HOSTNAME"
else
URL_HOSTNAME=localhost
URL_PATHNAME=$CHECK_URL
fi
# This URL will show up in the messages
LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME"
# And how we formulate the CURL request
CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"
dokku_log_verbose "CHECKS expected result:"
dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
[[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"
# Capture HTTP response or CURL error message
if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then
# OUTPUT contains the HTTP response
if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then
dokku_log_fail "$LOG_URL: expected to but did not find: \"$EXPECTED\""
exit 1
until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]
do
FAILEDCHECKS=0
ATTEMPT=$(( ATTEMPT + 1 ))
dokku_log_info1 "Attempt $ATTEMPT/$ATTEMPTS Waiting for $WAIT seconds ..."
sleep $WAIT
# -q do not use .curlrc (must come first)
# --compressed Test compression handled correctly
# --fail Fail on server errors (4xx, 5xx)
# --location Follow redirects
CURL_OPTIONS="-q --compressed --fail --location --max-time $TIMEOUT"
exec < "$FILENAME"
while read CHECK_URL EXPECTED ; do
# Ignore empty lines and lines starting with #
# shellcheck disable=SC1001
[[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue
# Ignore variables
[[ "$CHECK_URL" =~ ^.+= ]] && continue
if [[ "$CHECK_URL" =~ ^https?: ]] ; then
URL_PROTOCOL=${CHECK_URL%:*}
CHECK_URL=${CHECK_URL#*:}
else
URL_PROTOCOL="http"
fi
if [[ "$CHECK_URL" =~ ^//.+ ]] ; then
# To test a URL with specific host name, we still make request to localhost,
# but we set Host header to $SEND_HOST.
#
# The pattern is
# //SEND_HOST/PATHNAME
UNPREFIXED=${CHECK_URL#//}
URL_HOSTNAME=${UNPREFIXED%%/*}
URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME}
HEADERS="-H Host:$URL_HOSTNAME"
else
URL_HOSTNAME=localhost
URL_PATHNAME=$CHECK_URL
fi
# This URL will show up in the messages
LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME"
# And how we formulate the CURL request
CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"
dokku_log_verbose "CHECKS expected result:"
dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
[[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"
# Capture HTTP response or CURL error message
if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then
# OUTPUT contains the HTTP response
if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then
dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
FAILEDCHECKS=$(( FAILEDCHECKS + 1 ))
fi
else
# Failed to connect/no response, OUTPUT contains error message
dokku_log_warn "$OUTPUT"
FAILEDCHECKS=$(( FAILEDCHECKS + 1 ))
fi
done
if [ $FAILEDCHECKS -gt 0 ]; then
dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed."
SUCCESS=0
else
# Failed to connect/no response, OUTPUT contains error message
dokku_log_fail "$OUTPUT"
exit 2
SUCCESS=1
fi
done
if [ $FAILEDCHECKS -gt 0 ]; then
dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks."
exit 1
fi
dokku_log_info1 "All checks successful!"