Add RETRIES feature to checks.

The system will loop through checks RETRIES times until they succeed.
if they do not eventually succeed, it will fail the deployment

plugins/checks/check-deploy

@@ -24,11 +24,14 @@
 # The default behavior is to wait for 5 seconds before running the first check,
 # and timeout each check to 30 minutes.
 #
-# You can change these by setting WAIT and TIMEOUT to different values, for
+# By default, checks will be retried 5 times.  
+
+# You can change these by setting WAIT, TIMEOUT and RETRIES to different values, for
 # example:
 #
 #   WAIT=30     # Wait 1/2 minute
 #   TIMEOUT=60  # Timeout after a minute
+#   RETRIES=10  # retry checks 10 times
 #
 
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
@@ -53,6 +56,8 @@ fi
 WAIT="${DOKKU_CHECKS_WAIT:-5}"
 # Wait this many seconds (default 30) for each response.
 TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
+# use this number of retries for checks
+RETRIES="${DOKKU_CHECKS_RETRIES:-5}"
 
 # try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
 # docker cp exits with status 1 when run as non-root user when it tries to chown the file
@@ -95,11 +100,17 @@ while read LINE ; do
     VALUE=${TRIM#*=}
     [[ "$NAME" = "WAIT" ]]    && WAIT=$VALUE
     [[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE
+    [[ "$NAME" = "RETRIES" ]] && RETRIES=$VALUE
   fi
 done
 
+UNCHECKED=1
 
-dokku_log_info1 "Waiting for $WAIT seconds ..."
+for (( i=1; i <= RETRIES && UNCHECKED>0 ; i++ ))
+do
+UNCHECKED=0
+
+dokku_log_info1 "Attempt $i/$RETRIES Waiting for $WAIT seconds ..."
 sleep $WAIT
 
 
@@ -154,14 +165,29 @@ while read CHECK_URL EXPECTED ; do
   if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then
     # OUTPUT contains the HTTP response
     if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then
-      dokku_log_fail "$LOG_URL: expected to but did not find: \"$EXPECTED\""
-      exit 1
+      dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
+      #exit 1
+      UNCHECKED=$(expr $UNCHECKED + 1 )
+
     fi
   else
     # Failed to connect/no response, OUTPUT contains error message
-    dokku_log_fail "$OUTPUT"
-    exit 2
+    dokku_log_warn "$OUTPUT"
+    #exit 2
+    UNCHECKED=$(expr $UNCHECKED + 1 )
+  
   fi
 done
 
+if [ $UNCHECKED -gt 0 ]; then
+    dokku_log_warn "Check attempt $i/$RETRIES failed.  Retrying..."
+    continue;
+fi
+done
+
+if [ $UNCHECKED -gt 0 ]; then
+    dokku_log_fail "Could not start due to $UNCHECKED failed checks."
+    exit 1
+fi
+
 dokku_log_info1 "All checks successful!"