Add RETRIES feature to checks.

The system will loop through checks RETRIES times until they succeed.
if they do not eventually succeed, it will fail the deployment
This commit is contained in:
Josh Cohen committed 2015-03-25 03:17:44 +00:00
1 parent 2949b620d8
commit 3b252d43c6
1 file changed
+32 -6
+32 -6
View File
@@ -24,11 +24,14 @@
# The default behavior is to wait for 5 seconds before running the first check, # The default behavior is to wait for 5 seconds before running the first check,
# and timeout each check to 30 minutes. # and timeout each check to 30 minutes.
# #
# You can change these by setting WAIT and TIMEOUT to different values, for # By default, checks will be retried 5 times.
# You can change these by setting WAIT, TIMEOUT and RETRIES to different values, for
# example: # example:
# #
# WAIT=30 # Wait 1/2 minute # WAIT=30 # Wait 1/2 minute
# TIMEOUT=60 # Timeout after a minute # TIMEOUT=60 # Timeout after a minute
# RETRIES=10 # retry checks 10 times
# #
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
@@ -53,6 +56,8 @@ fi
WAIT="${DOKKU_CHECKS_WAIT:-5}" WAIT="${DOKKU_CHECKS_WAIT:-5}"
# Wait this many seconds (default 30) for each response. # Wait this many seconds (default 30) for each response.
TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}" TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
# use this number of retries for checks
RETRIES="${DOKKU_CHECKS_RETRIES:-5}"
# try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist # try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
# docker cp exits with status 1 when run as non-root user when it tries to chown the file # docker cp exits with status 1 when run as non-root user when it tries to chown the file
@@ -95,11 +100,17 @@ while read LINE ; do
VALUE=${TRIM#*=} VALUE=${TRIM#*=}
[[ "$NAME" = "WAIT" ]] && WAIT=$VALUE [[ "$NAME" = "WAIT" ]] && WAIT=$VALUE
[[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE [[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE
[[ "$NAME" = "RETRIES" ]] && RETRIES=$VALUE
fi fi
done done
UNCHECKED=1
dokku_log_info1 "Waiting for $WAIT seconds ..." for (( i=1; i <= RETRIES && UNCHECKED>0 ; i++ ))
do
UNCHECKED=0
dokku_log_info1 "Attempt $i/$RETRIES Waiting for $WAIT seconds ..."
sleep $WAIT sleep $WAIT
@@ -154,14 +165,29 @@ while read CHECK_URL EXPECTED ; do
if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then
# OUTPUT contains the HTTP response # OUTPUT contains the HTTP response
if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then
dokku_log_fail "$LOG_URL: expected to but did not find: \"$EXPECTED\"" dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
exit 1 #exit 1
UNCHECKED=$(expr $UNCHECKED + 1 )
fi fi
else else
# Failed to connect/no response, OUTPUT contains error message # Failed to connect/no response, OUTPUT contains error message
dokku_log_fail "$OUTPUT" dokku_log_warn "$OUTPUT"
exit 2 #exit 2
UNCHECKED=$(expr $UNCHECKED + 1 )
fi fi
done done
if [ $UNCHECKED -gt 0 ]; then
dokku_log_warn "Check attempt $i/$RETRIES failed. Retrying..."
continue;
fi
done
if [ $UNCHECKED -gt 0 ]; then
dokku_log_fail "Could not start due to $UNCHECKED failed checks."
exit 1
fi
dokku_log_info1 "All checks successful!" dokku_log_info1 "All checks successful!"