Previously a crashed container would stay down, regardless of exit status. In some cases, it may be useful to restart the container. For example, an application may not be correctly implementing their error handling, or the crash may be caused by a transient error. By setting the restart policy to `on-failure:N` - where N is a number of max restarts - we can help developers guard against crashing applications. Note that this is not a replacement for proper error handling, nor does this include notifications to a developer when a container is restarted. Those patterns should be implemented application side, or via a feature request to docker. The value is configurable at the app-level by setting DOKKU_RESTART_LIMIT to a number. By default, containers will be restarted a max of 10 times. If a container crashes during the check-deploy plugin trigger, then the deploy will be marked as a failure. Closes #216 Closes #398 Closes #1327
214 lines
7.3 KiB
Bash
Executable File
214 lines
7.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Hook to check server against list of checks specified in CHECKS file.
|
|
#
|
|
# The CHECKS file may contain empty lines, comments (lines starting with #),
|
|
# settings (NAME=VALUE) and check instructions.
|
|
#
|
|
# The format of a check instruction is a path, optionally followed by the
|
|
# expected content. For example:
|
|
#
|
|
# / My Amazing App
|
|
# /stylesheets/index.css .body
|
|
# /scripts/index.js $(function()
|
|
# /images/logo.png
|
|
#
|
|
# To check an application that supports multiple hostnames, use relative URLs
|
|
# that include the hostname, for example:
|
|
#
|
|
# //admin.example.com Admin Dashboard
|
|
# //static.example.com/logo.png
|
|
#
|
|
# You can also specify the protocol to explicitly check HTTPS requests.
|
|
#
|
|
# The default behavior is to wait for 5 seconds before running the first check,
|
|
# and timeout each check to 30 seconds.
|
|
#
|
|
# By default, checks will be retried 5 times.
|
|
|
|
# You can change these by setting WAIT, TIMEOUT and ATTEMPTS to different values, for
|
|
# example:
|
|
#
|
|
# WAIT=30 # Wait 1/2 minute
|
|
# TIMEOUT=60 # Timeout after a minute
|
|
# ATTEMPTS=10 # retry checks 10 times
|
|
#
|
|
|
|
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
|
|
source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
|
|
source "$PLUGIN_AVAILABLE_PATH/config/functions"
|
|
|
|
APP="$1"; DOKKU_APP_CONTAINER_ID="$2"; DOKKU_APP_CONTAINER_TYPE="$3"; DOKKU_APP_LISTEN_PORT="$4"; DOKKU_APP_LISTEN_IP="$5"
|
|
if [[ -z "$DOKKU_APP_LISTEN_PORT" ]] && [[ -f "$DOKKU_ROOT/$APP/PORT" ]]; then
|
|
DOKKU_APP_LISTEN_PORT=$(< "$DOKKU_ROOT/$APP/PORT")
|
|
fi
|
|
if [[ -z "$DOKKU_APP_LISTEN_IP" ]] && [[ -f "$DOKKU_ROOT/$APP/IP" ]]; then
|
|
DOKKU_APP_LISTEN_IP=$(< "$DOKKU_ROOT/$APP/IP")
|
|
fi
|
|
if [[ -z "$DOKKU_APP_CONTAINER_ID" ]]; then
|
|
DOKKU_APP_CIDS=( $(get_app_container_ids $APP) )
|
|
DOKKU_APP_CONTAINER_ID=${DOKKU_APP_CIDS[0]}
|
|
fi
|
|
|
|
|
|
# source global and in-app envs to get DOKKU_CHECKS_WAIT and any other necessary vars
|
|
eval "$(config_export global)"
|
|
eval "$(config_export app $APP)"
|
|
|
|
# Wait this many seconds (default 5) for server to start before running checks.
|
|
WAIT="${DOKKU_CHECKS_WAIT:-5}"
|
|
# Wait this many seconds (default 30) for each response.
|
|
TIMEOUT="${DOKKU_CHECKS_TIMEOUT:-30}"
|
|
# use this number of retries for checks
|
|
ATTEMPTS="${DOKKU_CHECKS_ATTEMPTS:-5}"
|
|
|
|
# try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
|
|
# docker cp exits with status 1 when run as non-root user when it tries to chown the file
|
|
# after successfully copying the file. Thus, we suppress stderr.
|
|
# ref: https://github.com/dotcloud/docker/issues/3986
|
|
TMPDIR=$(mktemp -d /tmp/CHECKS.XXXXX)
|
|
docker cp $DOKKU_APP_CONTAINER_ID:/app/CHECKS $TMPDIR 2> /dev/null || true
|
|
|
|
FILENAME=${TMPDIR}/CHECKS
|
|
|
|
cleanup() {
|
|
rm -rf $TMPDIR
|
|
dokku_log_info2_quiet "$APP container output:"
|
|
dokku_container_log_verbose_quiet $DOKKU_APP_CONTAINER_ID
|
|
dokku_log_info2_quiet "end $APP container output"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [[ ! -s "${TMPDIR}/CHECKS" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]]; then
|
|
if [[ "$DOKKU_SKIP_DEFAULT_CHECKS" = "true" ]]; then
|
|
dokku_log_verbose "Skipping default check..."
|
|
exit 0
|
|
fi
|
|
dokku_log_verbose "For more efficient zero downtime deployments, create a file CHECKS."
|
|
dokku_log_verbose "See http://progrium.viewdocs.io/dokku/checks-examples.md for examples"
|
|
dokku_log_verbose "CHECKS file not found in container: Running simple container check..."
|
|
|
|
rm -rf $TMPDIR
|
|
|
|
# simple default check to see if the container stuck around
|
|
# for more thorough checks, create a CHECKS file
|
|
DOKKU_DEFAULT_CHECKS_WAIT="${DOKKU_DEFAULT_CHECKS_WAIT:-10}"
|
|
dokku_log_info1 "Waiting for $DOKKU_DEFAULT_CHECKS_WAIT seconds ..."
|
|
sleep $DOKKU_DEFAULT_CHECKS_WAIT
|
|
|
|
! (is_container_running $DOKKU_APP_CONTAINER_ID) && dokku_log_fail "App container failed to start!!"
|
|
container_restarts=$(docker inspect -f "{{ .RestartCount }}" $DOKKU_APP_CONTAINER_ID)
|
|
if [[ $container_restarts -ne 0 ]]; then
|
|
docker stop "$DOKKU_APP_CONTAINER_ID" || true
|
|
dokku_log_fail "App container failed to start!!"
|
|
fi
|
|
|
|
dokku_log_info1 "Default container check successful!" && exit 0
|
|
fi
|
|
|
|
# Reads name/value pairs, sets the WAIT and TIMEOUT variables
|
|
exec < "$FILENAME"
|
|
while read LINE; do
|
|
# Name/value pair
|
|
if [[ "$LINE" =~ ^.+= ]]; then
|
|
TRIM=${LINE%#*}
|
|
NAME=${TRIM%=*}
|
|
VALUE=${TRIM#*=}
|
|
[[ "$NAME" = "WAIT" ]] && WAIT=$VALUE
|
|
[[ "$NAME" = "TIMEOUT" ]] && TIMEOUT=$VALUE
|
|
[[ "$NAME" = "ATTEMPTS" ]] && ATTEMPTS=$VALUE
|
|
fi
|
|
done
|
|
|
|
ATTEMPT=0
|
|
|
|
until [[ $SUCCESS == 1 || $ATTEMPT -ge $ATTEMPTS ]]
|
|
do
|
|
FAILEDCHECKS=0
|
|
ATTEMPT=$(( ATTEMPT + 1 ))
|
|
dokku_log_info1 "Attempt $ATTEMPT/$ATTEMPTS Waiting for $WAIT seconds ..."
|
|
sleep $WAIT
|
|
|
|
|
|
# -q do not use .curlrc (must come first)
|
|
# --compressed Test compression handled correctly
|
|
# --fail Fail on server errors (4xx, 5xx)
|
|
# --location Follow redirects
|
|
CURL_OPTIONS="-q --compressed --fail --location --max-time $TIMEOUT"
|
|
|
|
# Set X-Forwarded-Proto header if TLS is enabled.
|
|
SSL="$DOKKU_ROOT/$APP/tls"; WILDCARD_SSL="$DOKKU_ROOT/tls"
|
|
if [[ -e "$SSL/server.crt" && -e "$SSL/server.key" ]] || [[ -e "$WILDCARD_SSL/server.crt" && -e "$WILDCARD_SSL/server.key" ]]; then
|
|
CURL_OPTIONS+=" -H X-Forwarded-Proto:https"
|
|
fi
|
|
|
|
exec < "$FILENAME"
|
|
while read CHECK_URL EXPECTED; do
|
|
# Ignore empty lines and lines starting with #
|
|
# shellcheck disable=SC1001
|
|
[[ -z "$CHECK_URL" || "$CHECK_URL" =~ ^\# ]] && continue
|
|
# Ignore if it's not a URL in a supported format
|
|
# shellcheck disable=SC1001
|
|
! [[ "$CHECK_URL" =~ ^(http(s)?:)?\/.* ]] && continue
|
|
|
|
if [[ "$CHECK_URL" =~ ^https?: ]]; then
|
|
URL_PROTOCOL=${CHECK_URL%:*}
|
|
CHECK_URL=${CHECK_URL#*:}
|
|
else
|
|
URL_PROTOCOL="http"
|
|
fi
|
|
|
|
if [[ "$CHECK_URL" =~ ^//.+ ]]; then
|
|
# To test a URL with specific host name, we still make request to localhost,
|
|
# but we set Host header to $SEND_HOST.
|
|
#
|
|
# The pattern is
|
|
# //SEND_HOST/PATHNAME
|
|
UNPREFIXED=${CHECK_URL#//}
|
|
URL_HOSTNAME=${UNPREFIXED%%/*}
|
|
URL_PATHNAME=${UNPREFIXED#$URL_HOSTNAME}
|
|
|
|
HEADERS="-H Host:$URL_HOSTNAME"
|
|
else
|
|
URL_HOSTNAME=localhost
|
|
URL_PATHNAME=$CHECK_URL
|
|
fi
|
|
|
|
# This URL will show up in the messages
|
|
LOG_URL="$URL_PROTOCOL://$URL_HOSTNAME$URL_PATHNAME"
|
|
# And how we formulate the CURL request
|
|
CURL_ARGS="$CURL_OPTIONS $URL_PROTOCOL://$DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT$URL_PATHNAME $HEADERS"
|
|
|
|
dokku_log_verbose "CHECKS expected result:"
|
|
dokku_log_verbose "$LOG_URL => \"$EXPECTED\""
|
|
[[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS"
|
|
|
|
# Capture HTTP response or CURL error message
|
|
if OUTPUT=$(curl -# $CURL_ARGS 2>&1); then
|
|
# OUTPUT contains the HTTP response
|
|
if [[ ! "$OUTPUT" =~ $EXPECTED ]]; then
|
|
dokku_log_warn "$LOG_URL: expected to but did not find: \"$EXPECTED\""
|
|
FAILEDCHECKS=$(( FAILEDCHECKS + 1 ))
|
|
fi
|
|
else
|
|
# Failed to connect/no response, OUTPUT contains error message
|
|
dokku_log_warn "$OUTPUT"
|
|
FAILEDCHECKS=$(( FAILEDCHECKS + 1 ))
|
|
fi
|
|
done
|
|
|
|
if [[ $FAILEDCHECKS -gt 0 ]]; then
|
|
dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed."
|
|
SUCCESS=0
|
|
else
|
|
SUCCESS=1
|
|
fi
|
|
done
|
|
|
|
if [[ $FAILEDCHECKS -gt 0 ]]; then
|
|
dokku_log_fail "Could not start due to $FAILEDCHECKS failed checks."
|
|
exit 1
|
|
fi
|
|
|
|
dokku_log_info1 "All checks successful!"
|