From 3b252d43c6329000cb21f76fc17e790a1f6ebd0d Mon Sep 17 00:00:00 2001
From: Josh Cohen <joshco@gmail.com>
Date: Wed, 25 Mar 2015 03:17:44 +0000
Subject: [PATCH] Add RETRIES feature to checks. The system will loop through
 checks RETRIES times until they succeed. if they do not eventually succeed,
 it will fail the deployment

---
 plugins/checks/check-deploy | 38 +++++++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy
index 861d90de9..a74bb3361 100755
--- a/plugins/checks/check-deploy
+++ b/plugins/checks/check-deploy
@@ -24,11 +24,14 @@
 # The default behavior is to wait for 5 seconds before running the first check,
 # and timeout each check to 30 minutes.
 #
-# You can change these by setting WAIT and TIMEOUT to different values, for
+# By default, checks will be retried 5 times.  
+
+# You can change these by setting WAIT, TIMEOUT and RETRIES to different values, for
 # example:
 #
 #   WAIT=30     # Wait 1/2 minute
 #   TIMEOUT=60  # Timeout after a minute
+#   RETRIES=10  # retry checks 10 times
 #
 
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
@@ -53,6 +56,8 @@ fi
 WAIT="${DOKKU_CHECKS_WAIT:-5}"
 # Wait this many seconds (default 30) for each response.
 TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
+# use this number of retries for checks
+RETRIES="${DOKKU_CHECKS_RETRIES:-5}"
 
 # try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
 # docker cp exits with status 1 when run as non-root user when it tries to chown the file
@@ -95,11 +100,17 @@ while read LINE ; do
     VALUE=${TRIM#*=}
     [[ "$NAME" = "WAIT" ]]    && WAIT=$VALUE
     [[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE
+    [[ "$NAME" = "RETRIES" ]] && RETRIES=$VALUE
   fi
 done
 
+UNCHECKED=1
 
-dokku_log_info1 "Waiting for $WAIT seconds ..."
+for (( i=1; i <= RETRIES && UNCHECKED>0 ; i++ ))
+do
+UNCHECKED=0
+
+dokku_log_info1 "Attempt $i/$RETRIES Waiting for $WAIT seconds ..."
 sleep $WAIT
 
 
@@ -154,14 +165,29 @@ while read CHECK_URL EXPECTED ; do
   if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then
     # OUTPUT contains the HTTP response
     if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then
-      dokku_log_fail "$LOG_URL: expected to but did not find: \"$EXPECTED\""
-      exit 1
+      dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
+      #exit 1
+      UNCHECKED=$(expr $UNCHECKED + 1 )
+
     fi
   else
     # Failed to connect/no response, OUTPUT contains error message
-    dokku_log_fail "$OUTPUT"
-    exit 2
+    dokku_log_warn "$OUTPUT"
+    #exit 2
+    UNCHECKED=$(expr $UNCHECKED + 1 )
+  
   fi
 done
 
+if [ $UNCHECKED -gt 0 ]; then
+    dokku_log_warn "Check attempt $i/$RETRIES failed.  Retrying..."
+    continue;
+fi
+done
+
+if [ $UNCHECKED -gt 0 ]; then
+    dokku_log_fail "Could not start due to $UNCHECKED failed checks."
+    exit 1
+fi
+
 dokku_log_info1 "All checks successful!"