Files
dokku/plugins/checks/check-deploy
Jose Diaz-Gonzalez ab8eb5a727 Handle crashing containers by using restart=on-failure policy
Previously a crashed container would stay down, regardless of exit status. In some cases, it may be useful to restart the container. For example, an application may not be correctly implementing their error handling, or the crash may be caused by a transient error.

By setting the restart policy to `on-failure:N` - where N is a number of max restarts - we can help developers guard against crashing applications. Note that this is not a replacement for proper error handling, nor does this include notifications to a developer when a container is restarted. Those patterns should be implemented application side, or via a feature request to docker.

The value is configurable at the app-level by setting DOKKU_RESTART_LIMIT to a number. By default, containers will be restarted a max of 10 times. If a container crashes during the check-deploy plugin trigger, then the deploy will be marked as a failure.

Closes #216
Closes #398
Closes #1327
2015-09-16 23:47:33 -07:00

214 lines
7.3 KiB
Bash
Executable File

#!/usr/bin/env bash
# Hook to check server against list of checks specified in CHECKS file.
#
# The CHECKS file may contain empty lines, comments (lines starting with #),
# settings (NAME=VALUE) and check instructions.
#
# The format of a check instruction is a path, optionally followed by the
# expected content. For example:
#
# / My Amazing App
# /stylesheets/index.css .body
# /scripts/index.js $(function()
# /images/logo.png
#
# To check an application that supports multiple hostnames, use relative URLs
# that include the hostname, for example:
#
# //admin.example.com Admin Dashboard
# //static.example.com/logo.png
#
# You can also specify the protocol to explicitly check HTTPS requests.
#
# The default behavior is to wait for 5 seconds before running the first check,
# and timeout each check to 30 seconds.
#
# By default, checks will be retried 5 times.
# You can change these by setting WAIT, TIMEOUT and ATTEMPTS to different values, for
# example:
#
# WAIT=30 # Wait 1/2 minute
# TIMEOUT=60 # Timeout after a minute
# ATTEMPTS=10 # retry checks 10 times
#
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
source "$PLUGIN_AVAILABLE_PATH/config/functions"
APP="$1"; DOKKU_APP_CONTAINER_ID="$2"; DOKKU_APP_CONTAINER_TYPE="$3"; DOKKU_APP_LISTEN_PORT="$4"; DOKKU_APP_LISTEN_IP="$5"
if [[ -z "$DOKKU_APP_LISTEN_PORT" ]] && [[ -f "$DOKKU_ROOT/$APP/PORT" ]]; then
DOKKU_APP_LISTEN_PORT=$(< "$DOKKU_ROOT/$APP/PORT")
fi
if [[ -z "$DOKKU_APP_LISTEN_IP" ]] && [[ -f "$DOKKU_ROOT/$APP/IP" ]]; then
DOKKU_APP_LISTEN_IP=$(< "$DOKKU_ROOT/$APP/IP")
fi
if [[ -z "$DOKKU_APP_CONTAINER_ID" ]]; then
DOKKU_APP_CIDS=( $(get_app_container_ids $APP) )
DOKKU_APP_CONTAINER_ID=${DOKKU_APP_CIDS[0]}
fi
# source global and in-app envs to get DOKKU_CHECKS_WAIT and any other necessary vars
eval "$(config_export global)"
eval "$(config_export app $APP)"
# Wait this many seconds (default 5) for server to start before running checks.
WAIT="${DOKKU_CHECKS_WAIT:-5}"
# Wait this many seconds (default 30) for each response.
TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
# use this number of retries for checks
ATTEMPTS="${DOKKU_CHECKS_ATTEMPTS:-5}"
# try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
# docker cp exits with status 1 when run as non-root user when it tries to chown the file
# after successfully copying the file. Thus, we suppress stderr.
# ref: https://github.com/dotcloud/docker/issues/3986
TMPDIR=$(mktemp -d /tmp/CHECKS.XXXXX)
docker cp $DOKKU_APP_CONTAINER_ID:/app/CHECKS $TMPDIR 2> /dev/null || true
FILENAME=${TMPDIR}/CHECKS
cleanup() {
rm -rf $TMPDIR
dokku_log_info2_quiet "$APP container output:"
dokku_container_log_verbose_quiet $DOKKU_APP_CONTAINER_ID
dokku_log_info2_quiet "end $APP container output"
}
trap cleanup EXIT
if [[ ! -s "${TMPDIR}/CHECKS" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]]; then
if [[ "$DOKKU_SKIP_DEFAULT_CHECKS" = "true" ]]; then
dokku_log_verbose "Skipping default check..."
exit 0
fi
dokku_log_verbose "For more efficient zero downtime deployments, create a file CHECKS."
dokku_log_verbose "See http://progrium.viewdocs.io/dokku/checks-examples.md for examples"
dokku_log_verbose "CHECKS file not found in container: Running simple container check..."
rm -rf $TMPDIR
# simple default check to see if the container stuck around
# for more thorough checks, create a CHECKS file
DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}"
dokku_log_info1 "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ..."
sleep $DOKKU_DEFAULT_CHECKS_WAIT
! (is_container_running $DOKKU_APP_CONTAINER_ID) && dokku_log_fail "App container failed to start!!"
container_restarts=$(docker inspect -f "{{ .RestartCount }}" $DOKKU_APP_CONTAINER_ID)
if [[ $container_restarts -ne 0 ]]; then
docker stop "$DOKKU_APP_CONTAINER_ID" || true
dokku_log_fail "App container failed to start!!"
fi
dokku_log_info1 "Default container check successful!" && exit 0
fi
# Reads name/value pairs, sets the WAIT and TIMEOUT variables
exec < "$FILENAME"
while read LINE; do
# Name/value pair
if [[ "$LINE" =~ ^.+= ]]; then
TRIM=${LINE%#*}
NAME=${TRIM%=*}
VALUE=${TRIM#*=}
[[ "$NAME" = "WAIT" ]] && WAIT=$VALUE
[[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE
[[ "$NAME" = "ATTEMPTS" ]] && ATTEMPTS=$VALUE
fi
done
ATTEMPT=0
until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]
do
FAILEDCHECKS=0
ATTEMPT=$(( ATTEMPT + 1 ))
dokku_log_info1 "Attempt $ATTEMPT/$ATTEMPTS Waiting for $WAIT seconds ..."
sleep $WAIT
# -q do not use .curlrc (must come first)
# --compressed Test compression handled correctly
# --fail Fail on server errors (4xx, 5xx)
# --location Follow redirects
CURL_OPTIONS="-q --compressed --fail --location --max-time $TIMEOUT"
# Set X-Forwarded-Proto header if TLS is enabled.
SSL="$DOKKU_ROOT/$APP/tls"; WILDCARD_SSL="$DOKKU_ROOT/tls"
if [[ -e "$SSL/server.crt" && -e "$SSL/server.key" ]] || [[ -e "$WILDCARD_SSL/server.crt" && -e "$WILDCARD_SSL/server.key" ]]; then
CURL_OPTIONS+=" -H X-Forwarded-Proto:https"
fi
exec < "$FILENAME"
while read CHECK_URL EXPECTED; do
# Ignore empty lines and lines starting with #
# shellcheck disable=SC1001
[[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue
# Ignore if it's not a URL in a supported format
# shellcheck disable=SC1001
! [[ "$CHECK_URL" =~ ^(http(s)?:)?\/.* ]] && continue
if [[ "$CHECK_URL" =~ ^https?: ]]; then
URL_PROTOCOL=${CHECK_URL%:*}
CHECK_URL=${CHECK_URL#*:}
else
URL_PROTOCOL="http"
fi
if [[ "$CHECK_URL" =~ ^//.+ ]]; then
# To test a URL with specific host name, we still make request to localhost,
# but we set Host header to $SEND_HOST.
#
# The pattern is
# //SEND_HOST/PATHNAME
UNPREFIXED=${CHECK_URL#//}
URL_HOSTNAME=${UNPREFIXED%%/*}
URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME}
HEADERS="-H Host:$URL_HOSTNAME"
else
URL_HOSTNAME=localhost
URL_PATHNAME=$CHECK_URL
fi
# This URL will show up in the messages
LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME"
# And how we formulate the CURL request
CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"
dokku_log_verbose "CHECKS expected result:"
dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
[[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"
# Capture HTTP response or CURL error message
if OUTPUT=$(curl -# $CURL_ARGS 2>&1); then
# OUTPUT contains the HTTP response
if [[ ! "$OUTPUT" =~ $EXPECTED ]]; then
dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
FAILEDCHECKS=$(( FAILEDCHECKS + 1 ))
fi
else
# Failed to connect/no response, OUTPUT contains error message
dokku_log_warn "$OUTPUT"
FAILEDCHECKS=$(( FAILEDCHECKS + 1 ))
fi
done
if [[ $FAILEDCHECKS -gt 0 ]]; then
dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed."
SUCCESS=0
else
SUCCESS=1
fi
done
if [[ $FAILEDCHECKS -gt 0 ]]; then
dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks."
exit 1
fi
dokku_log_info1 "All checks successful!"