diff --git a/docs/deployment/logs.md b/docs/deployment/logs.md index eefb5983c..f36deceac 100644 --- a/docs/deployment/logs.md +++ b/docs/deployment/logs.md @@ -1,19 +1,21 @@ # Log Management ``` -logs # display recent log output +logs [-h] [-t] [-n num] [-q] [-p process] # Display recent log output +logs:failed # Shows the last failed deploy logs for an app ``` ## Usage -### Application Logs -You can easily get logs of an application using the `logs` command: +### App Logs + +You can easily get logs of an app using the `logs` command: ```shell dokku logs node-js-app ``` -## Behavioral modifiers +#### Behavioral modifiers Dokku also supports certain command-line arguments that augment the `log` command's behavior. @@ -26,8 +28,19 @@ Dokku also supports certain command-line arguments that augment the `log` comman You can use these modifiers as follows: - ```shell dokku logs node-js-app -t -p web ``` will show logs continually from the web process. + +### Failed Deploy Logs + +> Warning: The default `docker-local` scheduler will "store" these until the next deploy or until the old containers are garbage collected - whichever runs first. If you require the logs beyond this point in time, please ship the logs to a centralized log server. + +In some cases, it may be useful to retrieve the logs from a previously failed deploy. + +You can retrieve these logs by using the `logs:failed` command + +```shell +dokku logs:failed node-js-app +``` diff --git a/dokku b/dokku index 21c3588c0..96222c23f 100755 --- a/dokku +++ b/dokku @@ -42,6 +42,8 @@ export PLUGIN_ENABLED_PATH=${PLUGIN_ENABLED_PATH:="$PLUGIN_PATH/enabled"} export PLUGIN_CORE_PATH=${PLUGIN_CORE_PATH:="$DOKKU_LIB_ROOT/core-plugins"} export PLUGIN_CORE_AVAILABLE_PATH=${PLUGIN_CORE_AVAILABLE_PATH:="$PLUGIN_CORE_PATH/available"} export PLUGIN_CORE_ENABLED_PATH=${PLUGIN_CORE_ENABLED_PATH:="$PLUGIN_CORE_PATH/enabled"} +export DOKKU_SYSTEM_GROUP=${DOKKU_SYSTEM_GROUP:="dokku"} +export DOKKU_SYSTEM_USER=${DOKKU_SYSTEM_USER:="dokku"} export DOKKU_API_VERSION=1 export DOKKU_NOT_IMPLEMENTED_EXIT=10 diff --git a/plugins/common/property-functions b/plugins/common/property-functions index f995ce6ef..ff68008f1 100755 --- a/plugins/common/property-functions +++ b/plugins/common/property-functions @@ -67,15 +67,6 @@ fn-plugin-property-setup() { mkdir -p "${DOKKU_LIB_ROOT}/config/${PLUGIN}" chmod 0755 "${DOKKU_LIB_ROOT}/config/${PLUGIN}" - - if [[ -z "$DOKKU_SYSTEM_GROUP" ]]; then - export DOKKU_SYSTEM_GROUP=dokku - fi - - if [[ -z "$DOKKU_SYSTEM_USER" ]]; then - export DOKKU_SYSTEM_USER=dokku - fi - chown "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/config/${PLUGIN}" } @@ -86,7 +77,7 @@ fn-plugin-property-read() { if [[ -z "$PLUGIN" ]] || [[ -z "$APP" ]] || [[ -z "$KEY" ]]; then return 1 fi - + if [[ ! -f "${DOKKU_LIB_ROOT:?}/config/${PLUGIN}/${APP}/${KEY}" ]]; then return 1 fi diff --git a/plugins/logs/commands b/plugins/logs/commands index f51a3f750..9a3d7f7a5 100755 --- a/plugins/logs/commands +++ b/plugins/logs/commands @@ -1,27 +1,11 @@ #!/usr/bin/env bash [[ " help logs:help " == *" $1 "* ]] || exit "$DOKKU_NOT_IMPLEMENTED_EXIT" +source "$PLUGIN_AVAILABLE_PATH/logs/internal-functions" set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -usage() { - declare desc="logs specific usage" - echo "Usage: dokku logs " - echo " display recent log output" - echo "" - echo " -n, --num NUM # the number of lines to display" - echo " -p, --ps PS # only display logs from the given process" - echo " -t, --tail # continually stream logs" - echo " -q, --quiet # display raw logs without colors, time and names" -} - case "$1" in - logs:help) - usage - ;; - - help) - cat< [-h] [-t] [-n num] [-q] [-p process], Show the last logs for an application -EOF + help | logs:help) + cmd-logs-help "$@" ;; *) diff --git a/plugins/logs/internal-functions b/plugins/logs/internal-functions new file mode 100755 index 000000000..81aeb6ce5 --- /dev/null +++ b/plugins/logs/internal-functions @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +fn-logs-usage() { + declare desc="logs specific usage" + echo "Usage: dokku logs " + echo " display recent log output" + echo "" + echo " -n, --num NUM # the number of lines to display" + echo " -p, --ps PS # only display logs from the given process" + echo " -t, --tail # continually stream logs" + echo " -q, --quiet # display raw logs without colors, time and names" +} + +fn-logs-help-content() { + declare desc="return logs plugin help content" + cat< [-h] [-t] [-n num] [-q] [-p process], Display recent log output + logs:failed , Show the last logs for an application +help_content +} + +cmd-logs-help() { + if [[ $1 = "logs:help" ]] ; then + echo -e 'Usage: dokku logs[:COMMAND]' + echo '' + echo 'Output app logs' + echo '' + echo 'Additional commands:' + fn-logs-help-content | sort | column -c2 -t -s, + elif [[ $(ps -o command= $PPID) == *"--all"* ]]; then + fn-logs-help-content + else + cat<&2 ; exit 1 ; fi + if [[ "$EXIT_CODE" != 0 ]]; then fn-logs-usage >&2 ; exit 1 ; fi eval set -- "$TEMP" local DOKKU_LOGS_LINE_NUMBERS="100" @@ -29,7 +27,7 @@ logs_default_cmd() { -p|--ps) local DOKKU_LOGS_ONLY_PROCESS="$2"; shift 2 ;; -q|--quiet) local DOKKU_LOGS_NO_PRETTY_PRINT="true"; shift ;; --) shift; break ;; - *) echo "Internal error"; exit 1;; + *) dokku_log_fail "Internal error";; esac done diff --git a/plugins/logs/subcommands/failed b/plugins/logs/subcommands/failed new file mode 100755 index 000000000..1071a218d --- /dev/null +++ b/plugins/logs/subcommands/failed @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_AVAILABLE_PATH/config/functions" + +cmd-logs-failed() { + declare desc="shows the last failed deploy logs for an app" + declare cmd="logs:failed" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1 + declare APP="$1" + local DOKKU_SCHEDULER + [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on" + + DOKKU_SCHEDULER=$(config_get "$APP" DOKKU_SCHEDULER || echo "docker-local") + plugn trigger scheduler-failed "$DOKKU_SCHEDULER" "$APP" +} + +cmd-logs-failed "$@" diff --git a/plugins/scheduler-docker-local/install b/plugins/scheduler-docker-local/install new file mode 100755 index 000000000..a6159c0b5 --- /dev/null +++ b/plugins/scheduler-docker-local/install @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +scheduler-docker-local-install() { + declare desc="scheduler-docker-local install plugin trigger" + declare trigger="scheduler-docker-local-install" + + mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" +} + +scheduler-docker-local-install "$@" diff --git a/plugins/scheduler-docker-local/post-create b/plugins/scheduler-docker-local/post-create new file mode 100755 index 000000000..9c9f98c29 --- /dev/null +++ b/plugins/scheduler-docker-local/post-create @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +scheduler-docker-local-post-create() { + declare desc="scheduler-docker-local post-create plugin trigger" + declare trigger="scheduler-docker-local-post-create" + declare APP="$1" + + mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/${APP}" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/${APP}" +} + +scheduler-docker-local-post-create "$@" diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index e4dd5bb78..c4a463ae6 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -13,6 +13,8 @@ scheduler-docker-local-scheduler-deploy() { return fi + rm -f "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers" + local DOKKU_DOCKER_STOP_TIMEOUT DOKKU_HEROKUISH DOKKU_NETWORK_BIND_ALL IMAGE DOKKU_HEROKUISH=false IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") @@ -96,14 +98,16 @@ scheduler-docker-local-scheduler-deploy() { kill_new() { declare desc="wrapper function to kill newly started app container" - local cid="$1" - docker inspect "$cid" &> /dev/null && docker stop "$cid" > /dev/null && docker kill "$cid" &> /dev/null + declare CID="$1" PROC_TYPE="$2" CONTAINER_INDEX="$3" + docker inspect "$CID" &> /dev/null && docker stop "$CID" > /dev/null && docker kill "$CID" &> /dev/null + mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP" + echo "${CID} ${PROC_TYPE}.${CONTAINER_INDEX}" >> "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers" trap - INT TERM EXIT kill -9 $$ } # run checks first, then post-deploy hooks, which switches proxy traffic - trap 'kill_new $cid' INT TERM EXIT + trap 'kill_new $cid $PROC_TYPE $CONTAINER_INDEX' INT TERM EXIT if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "false" ]]; then dokku_log_info1 "Attempting pre-flight checks" plugn trigger check-deploy "$APP" "$cid" "$PROC_TYPE" "$port" "$ipaddr" diff --git a/plugins/scheduler-docker-local/scheduler-failed b/plugins/scheduler-docker-local/scheduler-failed new file mode 100755 index 000000000..cbba2b9c6 --- /dev/null +++ b/plugins/scheduler-docker-local/scheduler-failed @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" + +scheduler-docker-local-scheduler-failed() { + declare desc="scheduler-docker-local scheduler-failed plugin trigger" + declare trigger="scheduler-docker-local scheduler-failed" + declare DOKKU_SCHEDULER="$1" APP="$2" + local FAILED_CONTAINERS_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers" + local DEAD_CONTAINERS=() DOKKU_LOGS_CMD=() RUNNING_CONTAINERS=() + local CID LINE PREFIX + shift 2 + + if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then + return + fi + + if [[ ! -f "$FAILED_CONTAINERS_FILE" ]]; then + dokku_log_warn "No failed containers found" + return + fi + + while read -r LINE || [[ -n "$LINE" ]]; do + CID="$(echo "$LINE" | cut -d ' ' -f1)" + PREFIX="$(echo "$LINE" | cut -d ' ' -f2)" + if docker inspect "${CID}" > /dev/null 2>&1; then + RUNNING_CONTAINERS+=("$CID") + else + DEAD_CONTAINERS+=("$CID") + fi + done < "$FAILED_CONTAINERS_FILE" + + for CID in "${DEAD_CONTAINERS[@]}"; do + dokku_log_warn "App container $CID no longer running" + sed -i "/${CID}/d" "$FAILED_CONTAINERS_FILE" + done + + if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then + dokku_log_warn "No failed containers found" + return + fi + + ((MAX_INDEX=${#RUNNING_CONTAINERS[*]} - 1)) || true + for i in ${!RUNNING_CONTAINERS[*]}; do + local CID="${RUNNING_CONTAINERS[i]}" + DOKKU_LOGS_CMD+="(docker logs $DOKKU_LOGS_ARGS $CID 2>&1)" + if [[ $i != "$MAX_INDEX" ]]; then + local DOKKU_LOGS_CMD+="& " + else + local DOKKU_LOGS_CMD+="; " + fi + done + bash -c "($DOKKU_LOGS_CMD)" +} + +scheduler-docker-local-scheduler-failed "$@"