From 3b252d43c6329000cb21f76fc17e790a1f6ebd0d Mon Sep 17 00:00:00 2001 From: Josh Cohen Date: Wed, 25 Mar 2015 03:17:44 +0000 Subject: [PATCH 1/9] Add RETRIES feature to checks. The system will loop through checks RETRIES times until they succeed. if they do not eventually succeed, it will fail the deployment --- plugins/checks/check-deploy | 38 +++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy index 861d90de9..a74bb3361 100755 --- a/plugins/checks/check-deploy +++ b/plugins/checks/check-deploy @@ -24,11 +24,14 @@ # The default behavior is to wait for 5 seconds before running the first check, # and timeout each check to 30 minutes. # -# You can change these by setting WAIT and TIMEOUT to different values, for +# By default, checks will be retried 5 times. + +# You can change these by setting WAIT, TIMEOUT and RETRIES to different values, for # example: # # WAIT=30 # Wait 1/2 minute # TIMEOUT=60 # Timeout after a minute +# RETRIES=10 # retry checks 10 times # set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x @@ -53,6 +56,8 @@ fi WAIT="${DOKKU_CHECKS_WAIT:-5}" # Wait this many seconds (default 30) for each response. TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}" +# use this number of retries for checks +RETRIES="${DOKKU_CHECKS_RETRIES:-5}" # try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist # docker cp exits with status 1 when run as non-root user when it tries to chown the file @@ -95,11 +100,17 @@ while read LINE ; do VALUE=${TRIM#*=} [[ "$NAME" = "WAIT" ]] && WAIT=$VALUE [[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE + [[ "$NAME" = "RETRIES" ]] && RETRIES=$VALUE fi done +UNCHECKED=1 -dokku_log_info1 "Waiting for $WAIT seconds ..." +for (( i=1; i <= RETRIES && UNCHECKED>0 ; i++ )) +do +UNCHECKED=0 + +dokku_log_info1 "Attempt $i/$RETRIES Waiting for $WAIT seconds ..." sleep $WAIT @@ -154,14 +165,29 @@ while read CHECK_URL EXPECTED ; do if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then # OUTPUT contains the HTTP response if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then - dokku_log_fail "$LOG_URL: expected to but did not find: \"$EXPECTED\"" - exit 1 + dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\"" + #exit 1 + UNCHECKED=$(expr $UNCHECKED + 1 ) + fi else # Failed to connect/no response, OUTPUT contains error message - dokku_log_fail "$OUTPUT" - exit 2 + dokku_log_warn "$OUTPUT" + #exit 2 + UNCHECKED=$(expr $UNCHECKED + 1 ) + fi done +if [ $UNCHECKED -gt 0 ]; then + dokku_log_warn "Check attempt $i/$RETRIES failed. Retrying..." + continue; +fi +done + +if [ $UNCHECKED -gt 0 ]; then + dokku_log_fail "Could not start due to $UNCHECKED failed checks." + exit 1 +fi + dokku_log_info1 "All checks successful!" From 48f76bcfef63ce978e89fbe69470819209cc774e Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 24 Mar 2015 23:26:31 -0400 Subject: [PATCH 2/9] Indentation Indentation for new outer loop --- plugins/checks/check-deploy | 146 ++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy index a74bb3361..10f7cacc0 100755 --- a/plugins/checks/check-deploy +++ b/plugins/checks/check-deploy @@ -108,83 +108,83 @@ UNCHECKED=1 for (( i=1; i <= RETRIES && UNCHECKED>0 ; i++ )) do -UNCHECKED=0 - -dokku_log_info1 "Attempt $i/$RETRIES Waiting for $WAIT seconds ..." -sleep $WAIT - - -# -q do not use .curlrc (must come first) -# --compressed Test compression handled correctly -# --fail Fail on server errors (4xx, 5xx) -# --location Follow redirects -CURL_OPTIONS="-q --compressed --fail --location --max-time $TIMEOUT" - - -exec < "$FILENAME" -while read CHECK_URL EXPECTED ; do - # Ignore empty lines and lines starting with # - # shellcheck disable=SC1001 - [[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue - # Ignore variables - [[ "$CHECK_URL" =~ ^.+= ]] && continue - - if [[ "$CHECK_URL" =~ ^https?: ]] ; then - URL_PROTOCOL=${CHECK_URL%:*} - CHECK_URL=${CHECK_URL#*:} - else - URL_PROTOCOL="http" - fi - - if [[ "$CHECK_URL" =~ ^//.+ ]] ; then - # To test a URL with specific host name, we still make request to localhost, - # but we set Host header to $SEND_HOST. - # - # The pattern is - # //SEND_HOST/PATHNAME - UNPREFIXED=${CHECK_URL#//} - URL_HOSTNAME=${UNPREFIXED%%/*} - URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME} - - HEADERS="-H Host:$URL_HOSTNAME" - else - URL_HOSTNAME=localhost - URL_PATHNAME=$CHECK_URL - fi - - # This URL will show up in the messages - LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME" - # And how we formulate the CURL request - CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS" - - dokku_log_verbose "CHECKS expected result:" - dokku_log_verbose "$LOG_URL => \"$EXPECTED\"" - [[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS" - - # Capture HTTP response or CURL error message - if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then - # OUTPUT contains the HTTP response - if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then - dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\"" - #exit 1 - UNCHECKED=$(expr $UNCHECKED + 1 ) - - fi - else - # Failed to connect/no response, OUTPUT contains error message - dokku_log_warn "$OUTPUT" - #exit 2 - UNCHECKED=$(expr $UNCHECKED + 1 ) + UNCHECKED=0 + dokku_log_info1 "Attempt $i/$RETRIES Waiting for $WAIT seconds ..." + sleep $WAIT + + + # -q do not use .curlrc (must come first) + # --compressed Test compression handled correctly + # --fail Fail on server errors (4xx, 5xx) + # --location Follow redirects + CURL_OPTIONS="-q --compressed --fail --location --max-time $TIMEOUT" + + + exec < "$FILENAME" + while read CHECK_URL EXPECTED ; do + # Ignore empty lines and lines starting with # + # shellcheck disable=SC1001 + [[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue + # Ignore variables + [[ "$CHECK_URL" =~ ^.+= ]] && continue + + if [[ "$CHECK_URL" =~ ^https?: ]] ; then + URL_PROTOCOL=${CHECK_URL%:*} + CHECK_URL=${CHECK_URL#*:} + else + URL_PROTOCOL="http" + fi + + if [[ "$CHECK_URL" =~ ^//.+ ]] ; then + # To test a URL with specific host name, we still make request to localhost, + # but we set Host header to $SEND_HOST. + # + # The pattern is + # //SEND_HOST/PATHNAME + UNPREFIXED=${CHECK_URL#//} + URL_HOSTNAME=${UNPREFIXED%%/*} + URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME} + + HEADERS="-H Host:$URL_HOSTNAME" + else + URL_HOSTNAME=localhost + URL_PATHNAME=$CHECK_URL + fi + + # This URL will show up in the messages + LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME" + # And how we formulate the CURL request + CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS" + + dokku_log_verbose "CHECKS expected result:" + dokku_log_verbose "$LOG_URL => \"$EXPECTED\"" + [[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS" + + # Capture HTTP response or CURL error message + if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then + # OUTPUT contains the HTTP response + if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then + dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\"" + #exit 1 + UNCHECKED=$(expr $UNCHECKED + 1 ) + + fi + else + # Failed to connect/no response, OUTPUT contains error message + dokku_log_warn "$OUTPUT" + #exit 2 + UNCHECKED=$(expr $UNCHECKED + 1 ) + + fi + done + + if [ $UNCHECKED -gt 0 ]; then + dokku_log_warn "Check attempt $i/$RETRIES failed. Retrying..." + continue; fi done -if [ $UNCHECKED -gt 0 ]; then - dokku_log_warn "Check attempt $i/$RETRIES failed. Retrying..." - continue; -fi -done - if [ $UNCHECKED -gt 0 ]; then dokku_log_fail "Could not start due to $UNCHECKED failed checks." exit 1 From 4a0c0eab25a01b6296b0bd590a92b13d025af41f Mon Sep 17 00:00:00 2001 From: Josh Cohen Date: Thu, 26 Mar 2015 18:29:39 -0400 Subject: [PATCH 3/9] add documentation --- docs/application-deployment.md | 6 +- docs/checks-examples.md | 201 +++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 docs/checks-examples.md diff --git a/docs/application-deployment.md b/docs/application-deployment.md index 9327ca13e..ab8274028 100644 --- a/docs/application-deployment.md +++ b/docs/application-deployment.md @@ -107,10 +107,14 @@ A check is a relative URL and may be followed by expected content from the page, /about Our Amazing Team ``` -Dokku will wait `DOKKU_CHECKS_WAIT` seconds (default: `5`) before running the checks to give server time to start. For shorter/longer wait, change the `DOKKU_CHECKS_WAIT` environment variable. +Dokku will wait `DOKKU_CHECKS_WAIT` seconds (default: `5`) before running the checks to give server time to start. For shorter/longer wait, change the `DOKKU_CHECKS_WAIT` environment variable. This can be overridden in the CHECKS file by setting WAIT=nn. Dokku will wait `DOKKU_WAIT_TO_RETIRE` seconds (default: `60`) before stopping the old container such that no existing connections to it are dropped. +Dokku will retry the checks DOKKU_CHECKS_RETRIES times until the checks are successful or DOKKU_CHECKS_RETRIES is exceeded. In the latter case, the deployment is considered failed. This can be overridden in the CHECKS file by setting RETRIES=nn. + +See [checks-examples.md](checks-examples.md) for examples and output. + # Removing a deployed app SSH onto the server, then execute: diff --git a/docs/checks-examples.md b/docs/checks-examples.md new file mode 100644 index 000000000..e03fb5227 --- /dev/null +++ b/docs/checks-examples.md @@ -0,0 +1,201 @@ +# Checks Examples + +The CHECKS file may contain empty lines, comments (lines starting with #), +settings (NAME=VALUE) and check instructions. + +The format of a check instruction is a path, optionally followed by the +expected content. For example: + + / My Amazing App + /stylesheets/index.css .body + /scripts/index.js $(function() + /images/logo.png + +To check an application that supports multiple hostnames, use relative URLs +that include the hostname, for example: + + //admin.example.com Admin Dashboard + //static.example.com/logo.png + +You can also specify the protocol to explicitly check HTTPS requests. + +The default behavior is to wait for 5 seconds before running the first check, +and timeout each check to 30 minutes. + +By default, checks will be retried 5 times. + +You can change these by setting WAIT, TIMEOUT and RETRIES to different values, for +example: + + WAIT=30 # Wait 1/2 minute + TIMEOUT=60 # Timeout after a minute + RETRIES=10 # retry checks 10 times + +# Example: Successful Rails Deployment +In this example, a rails applicaiton is successfully deployed to dokku. The initial round of checks fails while the server is starting, but once it starts they succeed and the deployment is successful. +RETRIES is set to 6, but the third attempt succeeds. + +## CHECKS file + +```` +WAIT=5 +RETRIES=6 +/check.txt simple_check +```` + +> check.txt is a text file returning the string 'simple_check' + +## Deploy Output + +```` +git push dokku master + +-----> Cleaning up... +-----> Building tcfpledge2 from buildstep... +-----> Adding BUILD_ENV to build environment... +-----> Ruby app detected +-----> Compiling Ruby/Rails +-----> Using Ruby version: ruby-2.0.0 + +..... + +-----> Discovering process types + Procfile declares types -> web +-----> Releasing myapp... +-----> Deploying myapp... +-----> Running pre-flight checks +-----> Attempt 1/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/check.txt => "simple_check" + ! +curl: (7) Failed to connect to 172.17.0.155 port 5000: Connection refused + ! Check attempt 1/6 failed. Retrying... +-----> Attempt 2/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/check.txt => "simple_check" + ! +curl: (7) Failed to connect to 172.17.0.155 port 5000: Connection refused + ! Check attempt 2/6 failed. Retrying... +-----> Attempt 3/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/check.txt => "simple_check" +-----> All checks successful! +=====> myapp container output: + => Booting Thin + => Rails 4.2.0 application starting in production on http://0.0.0.0:5000 + => Run `rails server -h` for more startup options + => Ctrl-C to shutdown server + Thin web server (v1.6.3 codename Protein Powder) + Maximum connections set to 1024 + Listening on 0.0.0.0:5000, CTRL+C to stop +=====> end myapp container output +-----> Running post-deploy +-----> Configuring myapp.dokku.example.com... +-----> Creating http nginx.conf +-----> Running nginx-pre-reload + Reloading nginx +-----> Shutting down old container in 60 seconds +=====> Application deployed: + http://myapp.dokku.example.com +```` + +# Example: Failing Rails Deployment +In this example, a Rails application fails to deploy. The reason for the failure is that the postgres database connection fails. The initial checks will fail while we wait for the server to start up, just like in the above example. However, once the server does start accepting connections, we will see an error 500 due to the postgres database connection failure. + +Once the retries have been exceeded, the deployment fails and we see the container output, which shows the Postgres connection errors. + +## CHECKS file + +```` +WAIT=5 +RETRIES=6 +/ +```` + +> The check to the root url '/' would normally access the database. + +## Deploy Output + +> Note: The output has been trimmed for brevity + +```` +git push dokku master + +-----> Cleaning up... +-----> Building tcfpledge2 from buildstep... +-----> Adding BUILD_ENV to build environment... +-----> Ruby app detected +-----> Compiling Ruby/Rails +-----> Using Ruby version: ruby-2.0.0 + +..... + +Discovering process types +Procfile declares types -> web +Releasing myapp... +Deploying myapp... +Running pre-flight checks +-----> Attempt 1/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/ => "" + ! +curl: (7) Failed to connect to 172.17.0.188 port 5000: Connection refused + ! Check attempt 1/6 failed. Retrying... +-----> Attempt 2/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/ => "" + ! +curl: (7) Failed to connect to 172.17.0.188 port 5000: Connection refused + ! Check attempt 2/6 failed. Retrying... +-----> Attempt 3/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/ => "" + ! +curl: (22) The requested URL returned error: 500 Internal Server Error + ! Check attempt 3/6 failed. Retrying... +-----> Attempt 4/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/ => "" + ! +curl: (22) The requested URL returned error: 500 Internal Server Error + ! Check attempt 4/6 failed. Retrying... +-----> Attempt 5/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/ => "" + ! +curl: (22) The requested URL returned error: 500 Internal Server Error + ! Check attempt 5/6 failed. Retrying... +-----> Attempt 6/6 Waiting for 5 seconds ... + CHECKS expected result: + http://localhost/ => "" + ! +curl: (22) The requested URL returned error: 500 Internal Server Error +Could not start due to 1 failed checks. + ! Check attempt 6/6 failed. Retrying... +=====> myapp container output: + => Booting Thin + => Rails 4.2.0 application starting in production on http://0.0.0.0:5000 + => Run `rails server -h` for more startup options + => Ctrl-C to shutdown server + Thin web server (v1.6.3 codename Protein Powder) + Maximum connections set to 1024 + Listening on 0.0.0.0:5000, CTRL+C to stop + Started GET "/" for 172.17.42.1 at 2015-03-26 21:36:47 +0000 + Is the server running on host "172.17.42.1" and accepting + TCP/IP connections on port 5431? + PG::ConnectionBad (could not connect to server: Connection refused + Is the server running on host "172.17.42.1" and accepting + TCP/IP connections on port 5431? + ): + vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:651:in `initialize' + vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:651:in `new' + vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:651:in `connect' + vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:242:in `initialize' + vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:44:in `new' + vendor/bundle/ruby/2.0.0/gems/activerecord-4.2.0/lib/active_record/connection_adapters/postgresql_adapter.rb:44:in `postgresql_connection +=====> end myapp container output +/usr/local/bin/dokku: line 49: 23409 Killed dokku deploy "$APP" +To dokku@dokku.example.com:myapp + ! [remote rejected] dokku -> master (pre-receive hook declined) +error: failed to push some refs to 'dokku@dokku.example.com:myapp' +```` \ No newline at end of file From f90c7b95249eff7fc00d4c9730423d013f8dbd1a Mon Sep 17 00:00:00 2001 From: Josh Cohen Date: Thu, 26 Mar 2015 18:32:50 -0400 Subject: [PATCH 4/9] docs tweak --- docs/checks-examples.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/checks-examples.md b/docs/checks-examples.md index e03fb5227..9e371f96b 100644 --- a/docs/checks-examples.md +++ b/docs/checks-examples.md @@ -47,6 +47,8 @@ RETRIES=6 ## Deploy Output +> Note: The output has been trimmed for brevity + ```` git push dokku master From b33aebcae28a5b8cb108d9c714c4a24de20caa9d Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 26 Mar 2015 19:13:45 -0400 Subject: [PATCH 5/9] sanitize example app name --- docs/checks-examples.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/checks-examples.md b/docs/checks-examples.md index 9e371f96b..6881fa644 100644 --- a/docs/checks-examples.md +++ b/docs/checks-examples.md @@ -53,7 +53,7 @@ RETRIES=6 git push dokku master -----> Cleaning up... ------> Building tcfpledge2 from buildstep... +-----> Building myapp from buildstep... -----> Adding BUILD_ENV to build environment... -----> Ruby app detected -----> Compiling Ruby/Rails @@ -124,7 +124,7 @@ RETRIES=6 git push dokku master -----> Cleaning up... ------> Building tcfpledge2 from buildstep... +-----> Building myapp from buildstep... -----> Adding BUILD_ENV to build environment... -----> Ruby app detected -----> Compiling Ruby/Rails @@ -200,4 +200,4 @@ Could not start due to 1 failed checks. To dokku@dokku.example.com:myapp ! [remote rejected] dokku -> master (pre-receive hook declined) error: failed to push some refs to 'dokku@dokku.example.com:myapp' -```` \ No newline at end of file +```` From 61270967c18babaad41bb3f1b5e68abd03873063 Mon Sep 17 00:00:00 2001 From: Josh Cohen Date: Tue, 7 Apr 2015 14:51:49 +0000 Subject: [PATCH 6/9] Change "RETRIES" to (MAX) "ATTEMPTS" as this better reflects behavior. ATTEMPTS=0 means don't run the checks. Useful if you want to skip the checks and can be overridden by DOKKU_CHECKS_ATTEMPTS ATTEMPTS=1 means run only once --- plugins/checks/check-deploy | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy index 10f7cacc0..7e5fdaeaa 100755 --- a/plugins/checks/check-deploy +++ b/plugins/checks/check-deploy @@ -26,12 +26,12 @@ # # By default, checks will be retried 5 times. -# You can change these by setting WAIT, TIMEOUT and RETRIES to different values, for +# You can change these by setting WAIT, TIMEOUT and ATTEMPTS to different values, for # example: # # WAIT=30 # Wait 1/2 minute # TIMEOUT=60 # Timeout after a minute -# RETRIES=10 # retry checks 10 times +# ATTEMPTS=10 # retry checks 10 times # set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x @@ -57,7 +57,7 @@ WAIT="${DOKKU_CHECKS_WAIT:-5}" # Wait this many seconds (default 30) for each response. TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}" # use this number of retries for checks -RETRIES="${DOKKU_CHECKS_RETRIES:-5}" +ATTEMPTS="${DOKKU_CHECKS_ATTEMPTS:-5}" # try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist # docker cp exits with status 1 when run as non-root user when it tries to chown the file @@ -100,17 +100,18 @@ while read LINE ; do VALUE=${TRIM#*=} [[ "$NAME" = "WAIT" ]] && WAIT=$VALUE [[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE - [[ "$NAME" = "RETRIES" ]] && RETRIES=$VALUE + [[ "$NAME" = "ATTEMPTS" ]] && ATTEMPTS=$VALUE fi done -UNCHECKED=1 +UNCHECKED=0 +ATTEMPT=0 -for (( i=1; i <= RETRIES && UNCHECKED>0 ; i++ )) +until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]] do UNCHECKED=0 - - dokku_log_info1 "Attempt $i/$RETRIES Waiting for $WAIT seconds ..." + ATTEMPT=$(expr $ATTEMPT + 1 ) + dokku_log_info1 "Attempt $ATTEMPT/$ATTEMPTS Waiting for $WAIT seconds ..." sleep $WAIT @@ -166,22 +167,22 @@ do # OUTPUT contains the HTTP response if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\"" - #exit 1 UNCHECKED=$(expr $UNCHECKED + 1 ) fi else # Failed to connect/no response, OUTPUT contains error message dokku_log_warn "$OUTPUT" - #exit 2 UNCHECKED=$(expr $UNCHECKED + 1 ) fi done if [ $UNCHECKED -gt 0 ]; then - dokku_log_warn "Check attempt $i/$RETRIES failed. Retrying..." - continue; + dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed." + SUCCESS=0 + else + SUCCESS=1 fi done From 4e6a980bc659c4e687b5f0eb849c42abc4ed0259 Mon Sep 17 00:00:00 2001 From: Josh Cohen Date: Tue, 7 Apr 2015 11:02:34 -0400 Subject: [PATCH 7/9] update documentation for ATTEMPTS change --- docs/application-deployment.md | 2 +- docs/checks-examples.md | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/application-deployment.md b/docs/application-deployment.md index ab8274028..08f934e0f 100644 --- a/docs/application-deployment.md +++ b/docs/application-deployment.md @@ -111,7 +111,7 @@ Dokku will wait `DOKKU_CHECKS_WAIT` seconds (default: `5`) before running the ch Dokku will wait `DOKKU_WAIT_TO_RETIRE` seconds (default: `60`) before stopping the old container such that no existing connections to it are dropped. -Dokku will retry the checks DOKKU_CHECKS_RETRIES times until the checks are successful or DOKKU_CHECKS_RETRIES is exceeded. In the latter case, the deployment is considered failed. This can be overridden in the CHECKS file by setting RETRIES=nn. +Dokku will retry the checks DOKKU_CHECKS_ATTEMPTS times until the checks are successful or DOKKU_CHECKS_ATTEMPTS is exceeded. In the latter case, the deployment is considered failed. This can be overridden in the CHECKS file by setting ATTEMPTS=nn. See [checks-examples.md](checks-examples.md) for examples and output. diff --git a/docs/checks-examples.md b/docs/checks-examples.md index 6881fa644..a77f56e37 100644 --- a/docs/checks-examples.md +++ b/docs/checks-examples.md @@ -22,24 +22,24 @@ You can also specify the protocol to explicitly check HTTPS requests. The default behavior is to wait for 5 seconds before running the first check, and timeout each check to 30 minutes. -By default, checks will be retried 5 times. +By default, checks will be attempted 5 times. (Retried 4 times) -You can change these by setting WAIT, TIMEOUT and RETRIES to different values, for +You can change these by setting WAIT, TIMEOUT and ATTEMPTS to different values, for example: WAIT=30 # Wait 1/2 minute TIMEOUT=60 # Timeout after a minute - RETRIES=10 # retry checks 10 times + ATTEMPTS=10 # attempt checks 10 times # Example: Successful Rails Deployment In this example, a rails applicaiton is successfully deployed to dokku. The initial round of checks fails while the server is starting, but once it starts they succeed and the deployment is successful. -RETRIES is set to 6, but the third attempt succeeds. +ATTEMPTS is set to 6, but the third attempt succeeds. ## CHECKS file ```` WAIT=5 -RETRIES=6 +ATTEMPTS=6 /check.txt simple_check ```` @@ -71,13 +71,13 @@ git push dokku master http://localhost/check.txt => "simple_check" ! curl: (7) Failed to connect to 172.17.0.155 port 5000: Connection refused - ! Check attempt 1/6 failed. Retrying... + ! Check attempt 1/6 failed. -----> Attempt 2/6 Waiting for 5 seconds ... CHECKS expected result: http://localhost/check.txt => "simple_check" ! curl: (7) Failed to connect to 172.17.0.155 port 5000: Connection refused - ! Check attempt 2/6 failed. Retrying... + ! Check attempt 2/6 failed. -----> Attempt 3/6 Waiting for 5 seconds ... CHECKS expected result: http://localhost/check.txt => "simple_check" @@ -104,13 +104,13 @@ curl: (7) Failed to connect to 172.17.0.155 port 5000: Connection refused # Example: Failing Rails Deployment In this example, a Rails application fails to deploy. The reason for the failure is that the postgres database connection fails. The initial checks will fail while we wait for the server to start up, just like in the above example. However, once the server does start accepting connections, we will see an error 500 due to the postgres database connection failure. -Once the retries have been exceeded, the deployment fails and we see the container output, which shows the Postgres connection errors. +Once the attempts have been exceeded, the deployment fails and we see the container output, which shows the Postgres connection errors. ## CHECKS file ```` WAIT=5 -RETRIES=6 +ATTEMPTS=6 / ```` @@ -142,38 +142,38 @@ Running pre-flight checks http://localhost/ => "" ! curl: (7) Failed to connect to 172.17.0.188 port 5000: Connection refused - ! Check attempt 1/6 failed. Retrying... + ! Check attempt 1/6 failed. -----> Attempt 2/6 Waiting for 5 seconds ... CHECKS expected result: http://localhost/ => "" ! curl: (7) Failed to connect to 172.17.0.188 port 5000: Connection refused - ! Check attempt 2/6 failed. Retrying... + ! Check attempt 2/6 failed. -----> Attempt 3/6 Waiting for 5 seconds ... CHECKS expected result: http://localhost/ => "" ! curl: (22) The requested URL returned error: 500 Internal Server Error - ! Check attempt 3/6 failed. Retrying... + ! Check attempt 3/6 failed. -----> Attempt 4/6 Waiting for 5 seconds ... CHECKS expected result: http://localhost/ => "" ! curl: (22) The requested URL returned error: 500 Internal Server Error - ! Check attempt 4/6 failed. Retrying... + ! Check attempt 4/6 failed. -----> Attempt 5/6 Waiting for 5 seconds ... CHECKS expected result: http://localhost/ => "" ! curl: (22) The requested URL returned error: 500 Internal Server Error - ! Check attempt 5/6 failed. Retrying... + ! Check attempt 5/6 failed. -----> Attempt 6/6 Waiting for 5 seconds ... CHECKS expected result: http://localhost/ => "" ! curl: (22) The requested URL returned error: 500 Internal Server Error Could not start due to 1 failed checks. - ! Check attempt 6/6 failed. Retrying... + ! Check attempt 6/6 failed. =====> myapp container output: => Booting Thin => Rails 4.2.0 application starting in production on http://0.0.0.0:5000 From cc565661e37ca294806f8d62611244b942f38aa8 Mon Sep 17 00:00:00 2001 From: Josh Cohen Date: Sat, 11 Apr 2015 01:10:14 +0000 Subject: [PATCH 8/9] fix linting issues rename UNCHECKED -> FAILEDCHECKS --- plugins/checks/check-deploy | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy index 7e5fdaeaa..88a34d164 100755 --- a/plugins/checks/check-deploy +++ b/plugins/checks/check-deploy @@ -104,13 +104,12 @@ while read LINE ; do fi done -UNCHECKED=0 ATTEMPT=0 until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]] do - UNCHECKED=0 - ATTEMPT=$(expr $ATTEMPT + 1 ) + FAILEDCHECKS=0 + ATTEMPT=$(( $ATTEMPT + 1 )) dokku_log_info1 "Attempt $ATTEMPT/$ATTEMPTS Waiting for $WAIT seconds ..." sleep $WAIT @@ -167,18 +166,16 @@ do # OUTPUT contains the HTTP response if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\"" - UNCHECKED=$(expr $UNCHECKED + 1 ) - + FAILEDCHECKS=$(( $FAILEDCHECKS + 1 )) fi else # Failed to connect/no response, OUTPUT contains error message dokku_log_warn "$OUTPUT" - UNCHECKED=$(expr $UNCHECKED + 1 ) - + FAILEDCHECKS=$(( $FAILEDCHECKS + 1 )) fi done - if [ $UNCHECKED -gt 0 ]; then + if [ $FAILEDCHECKS -gt 0 ]; then dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed." SUCCESS=0 else @@ -186,8 +183,8 @@ do fi done -if [ $UNCHECKED -gt 0 ]; then - dokku_log_fail "Could not start due to $UNCHECKED failed checks." +if [ $FAILEDCHECKS -gt 0 ]; then + dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks." exit 1 fi From b5eef489fd22dd656b5040cb94c49e4bd39fe7f7 Mon Sep 17 00:00:00 2001 From: Josh Cohen Date: Mon, 13 Apr 2015 20:22:14 +0000 Subject: [PATCH 9/9] linting fixes --- plugins/checks/check-deploy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy index 88a34d164..c91eef3b4 100755 --- a/plugins/checks/check-deploy +++ b/plugins/checks/check-deploy @@ -109,7 +109,7 @@ ATTEMPT=0 until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]] do FAILEDCHECKS=0 - ATTEMPT=$(( $ATTEMPT + 1 )) + ATTEMPT=$(( ATTEMPT + 1 )) dokku_log_info1 "Attempt $ATTEMPT/$ATTEMPTS Waiting for $WAIT seconds ..." sleep $WAIT @@ -166,12 +166,12 @@ do # OUTPUT contains the HTTP response if [[ ! "$OUTPUT" =~ $EXPECTED ]] ; then dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\"" - FAILEDCHECKS=$(( $FAILEDCHECKS + 1 )) + FAILEDCHECKS=$(( FAILEDCHECKS + 1 )) fi else # Failed to connect/no response, OUTPUT contains error message dokku_log_warn "$OUTPUT" - FAILEDCHECKS=$(( $FAILEDCHECKS + 1 )) + FAILEDCHECKS=$(( FAILEDCHECKS + 1 )) fi done