diff --git a/docs/advanced-usage/schedulers/docker-local.md b/docs/advanced-usage/schedulers/docker-local.md index 1427a0a97..2f9ee476a 100644 --- a/docs/advanced-usage/schedulers/docker-local.md +++ b/docs/advanced-usage/schedulers/docker-local.md @@ -25,7 +25,8 @@ This plugin implements various functionality through `plugn` triggers to integra - `pre-restore` - `scheduler-deploy` - `scheduler-docker-cleanup` +- `scheduler-logs-failed` - `scheduler-run` - `scheduler-stop` - `scheduler-tags-create` -- `scheduler-tags-destroy` \ No newline at end of file +- `scheduler-tags-destroy` diff --git a/docs/deployment/logs.md b/docs/deployment/logs.md index eefb5983c..656e0a858 100644 --- a/docs/deployment/logs.md +++ b/docs/deployment/logs.md @@ -1,19 +1,21 @@ # Log Management ``` -logs # display recent log output +logs [-h] [-t] [-n num] [-q] [-p process] # Display recent log output +logs:failed [] # Shows the last failed deploy logs ``` ## Usage -### Application Logs -You can easily get logs of an application using the `logs` command: +### App Logs + +You can easily get logs of an app using the `logs` command: ```shell dokku logs node-js-app ``` -## Behavioral modifiers +#### Behavioral modifiers Dokku also supports certain command-line arguments that augment the `log` command's behavior. @@ -26,8 +28,25 @@ Dokku also supports certain command-line arguments that augment the `log` comman You can use these modifiers as follows: - ```shell dokku logs node-js-app -t -p web ``` will show logs continually from the web process. + +### Failed Deploy Logs + +> Warning: The default `docker-local` scheduler will "store" these until the next deploy or until the old containers are garbage collected - whichever runs first. If you require the logs beyond this point in time, please ship the logs to a centralized log server. + +In some cases, it may be useful to retrieve the logs from a previously failed deploy. + +You can retrieve these logs by using the `logs:failed` command + +```shell +dokku logs:failed node-js-app +``` + +You can also retrieve the failed logs for each app: + +```shell +dokku logs:failed node-js-app +``` diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index a14bea974..57e79fbc7 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -1177,9 +1177,12 @@ APP="$1"; ### `scheduler-deploy` +> Warning: The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + - Description: Allows you to run scheduler commands when an app is deployed - Invoked by: `dokku deploy` -- Arguments: `$DOKKU_SCHEDULER APP IMAGE_TAG` +- Arguments: `$DOKKU_SCHEDULER $APP $IMAGE_TAG` - Example: ```shell @@ -1193,9 +1196,12 @@ DOKKU_SCHEDULER="$1"; APP="$2"; IMAGE_TAG="$3"; ### `scheduler-docker-cleanup` +> Warning: The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + - Description: Allows you to run scheduler commands when dokku cleanup is invoked - Invoked by: `dokku deploy, dokku cleanup` -- Arguments: `$DOKKU_SCHEDULER APP FORCE_CLEANUP` +- Arguments: `$DOKKU_SCHEDULER $APP $FORCE_CLEANUP` - Example: ```shell @@ -1207,11 +1213,33 @@ DOKKU_SCHEDULER="$1"; APP="$2"; FORCE_CLEANUP="$3"; # TODO ``` +### `scheduler-logs-failed` + +> Warning: The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + +- Description: Allows you to run scheduler commands when retrieving failed container logs +- Invoked by: `dokku logs:failed` +- Arguments: `$DOKKU_SCHEDULER $APP` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +DOKKU_SCHEDULER="$1"; APP="$2"; ARGS="${@:3}"; + +# TODO +``` + ### `scheduler-run` +> Warning: The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + - Description: Allows you to run scheduler commands when a command is executed for your app - Invoked by: `dokku run` -- Arguments: `$DOKKU_SCHEDULER APP ...ARGS` +- Arguments: `$DOKKU_SCHEDULER $APP ...ARGS` - Example: ```shell @@ -1225,9 +1253,12 @@ DOKKU_SCHEDULER="$1"; APP="$2"; ARGS="${@:3}"; ### `scheduler-stop` +> Warning: The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + - Description: Allows you to run scheduler commands when a tag is destroyed - Invoked by: `dokku apps:destroy, dokku ps:stop` -- Arguments: `$DOKKU_SCHEDULER APP REMOVE_CONTAINERS` +- Arguments: `$DOKKU_SCHEDULER $APP $REMOVE_CONTAINERS` - Example: ```shell @@ -1241,9 +1272,12 @@ DOKKU_SCHEDULER="$1"; APP="$2"; REMOVE_CONTAINERS="$3"; ### `scheduler-tags-create` +> Warning: The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + - Description: Allows you to run scheduler commands when a tag is created - Invoked by: `dokku tags:create` -- Arguments: `$DOKKU_SCHEDULER APP SOURCE_IMAGE TARGET_IMAGE` +- Arguments: `$DOKKU_SCHEDULER $APP $SOURCE_IMAGE $TARGET_IMAGE` - Example: ```shell @@ -1257,9 +1291,12 @@ DOKKU_SCHEDULER="$1" APP="$2" SOURCE_IMAGE="$3" TARGET_IMAGE="$4" ### `scheduler-tags-destroy` +> Warning: The scheduler plugin trigger apis are under development and may change +> between minor releases until the 1.0 release. + - Description: Allows you to run scheduler commands when a tag is destroyed - Invoked by: `dokku tags:destroy` -- Arguments: `$DOKKU_SCHEDULER APP IMAGE_REPO IMAGE_TAG` +- Arguments: `$DOKKU_SCHEDULER $APP $IMAGE_REPO $IMAGE_TAG` - Example: ```shell diff --git a/dokku b/dokku index 21c3588c0..96222c23f 100755 --- a/dokku +++ b/dokku @@ -42,6 +42,8 @@ export PLUGIN_ENABLED_PATH=${PLUGIN_ENABLED_PATH:="$PLUGIN_PATH/enabled"} export PLUGIN_CORE_PATH=${PLUGIN_CORE_PATH:="$DOKKU_LIB_ROOT/core-plugins"} export PLUGIN_CORE_AVAILABLE_PATH=${PLUGIN_CORE_AVAILABLE_PATH:="$PLUGIN_CORE_PATH/available"} export PLUGIN_CORE_ENABLED_PATH=${PLUGIN_CORE_ENABLED_PATH:="$PLUGIN_CORE_PATH/enabled"} +export DOKKU_SYSTEM_GROUP=${DOKKU_SYSTEM_GROUP:="dokku"} +export DOKKU_SYSTEM_USER=${DOKKU_SYSTEM_USER:="dokku"} export DOKKU_API_VERSION=1 export DOKKU_NOT_IMPLEMENTED_EXIT=10 diff --git a/plugins/common/property-functions b/plugins/common/property-functions index f995ce6ef..ff68008f1 100755 --- a/plugins/common/property-functions +++ b/plugins/common/property-functions @@ -67,15 +67,6 @@ fn-plugin-property-setup() { mkdir -p "${DOKKU_LIB_ROOT}/config/${PLUGIN}" chmod 0755 "${DOKKU_LIB_ROOT}/config/${PLUGIN}" - - if [[ -z "$DOKKU_SYSTEM_GROUP" ]]; then - export DOKKU_SYSTEM_GROUP=dokku - fi - - if [[ -z "$DOKKU_SYSTEM_USER" ]]; then - export DOKKU_SYSTEM_USER=dokku - fi - chown "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/config/${PLUGIN}" } @@ -86,7 +77,7 @@ fn-plugin-property-read() { if [[ -z "$PLUGIN" ]] || [[ -z "$APP" ]] || [[ -z "$KEY" ]]; then return 1 fi - + if [[ ! -f "${DOKKU_LIB_ROOT:?}/config/${PLUGIN}/${APP}/${KEY}" ]]; then return 1 fi diff --git a/plugins/logs/commands b/plugins/logs/commands index f51a3f750..9a3d7f7a5 100755 --- a/plugins/logs/commands +++ b/plugins/logs/commands @@ -1,27 +1,11 @@ #!/usr/bin/env bash [[ " help logs:help " == *" $1 "* ]] || exit "$DOKKU_NOT_IMPLEMENTED_EXIT" +source "$PLUGIN_AVAILABLE_PATH/logs/internal-functions" set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -usage() { - declare desc="logs specific usage" - echo "Usage: dokku logs " - echo " display recent log output" - echo "" - echo " -n, --num NUM # the number of lines to display" - echo " -p, --ps PS # only display logs from the given process" - echo " -t, --tail # continually stream logs" - echo " -q, --quiet # display raw logs without colors, time and names" -} - case "$1" in - logs:help) - usage - ;; - - help) - cat< [-h] [-t] [-n num] [-q] [-p process], Show the last logs for an application -EOF + help | logs:help) + cmd-logs-help "$@" ;; *) diff --git a/plugins/logs/internal-functions b/plugins/logs/internal-functions new file mode 100755 index 000000000..81aeb6ce5 --- /dev/null +++ b/plugins/logs/internal-functions @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +fn-logs-usage() { + declare desc="logs specific usage" + echo "Usage: dokku logs " + echo " display recent log output" + echo "" + echo " -n, --num NUM # the number of lines to display" + echo " -p, --ps PS # only display logs from the given process" + echo " -t, --tail # continually stream logs" + echo " -q, --quiet # display raw logs without colors, time and names" +} + +fn-logs-help-content() { + declare desc="return logs plugin help content" + cat< [-h] [-t] [-n num] [-q] [-p process], Display recent log output + logs:failed , Show the last logs for an application +help_content +} + +cmd-logs-help() { + if [[ $1 = "logs:help" ]] ; then + echo -e 'Usage: dokku logs[:COMMAND]' + echo '' + echo 'Output app logs' + echo '' + echo 'Additional commands:' + fn-logs-help-content | sort | column -c2 -t -s, + elif [[ $(ps -o command= $PPID) == *"--all"* ]]; then + fn-logs-help-content + else + cat<&2 ; exit 1 ; fi + if [[ "$EXIT_CODE" != 0 ]]; then fn-logs-usage >&2 ; exit 1 ; fi eval set -- "$TEMP" local DOKKU_LOGS_LINE_NUMBERS="100" @@ -29,7 +27,7 @@ logs_default_cmd() { -p|--ps) local DOKKU_LOGS_ONLY_PROCESS="$2"; shift 2 ;; -q|--quiet) local DOKKU_LOGS_NO_PRETTY_PRINT="true"; shift ;; --) shift; break ;; - *) echo "Internal error"; exit 1;; + *) dokku_log_fail "Internal error";; esac done diff --git a/plugins/logs/subcommands/failed b/plugins/logs/subcommands/failed new file mode 100755 index 000000000..a064ea6b7 --- /dev/null +++ b/plugins/logs/subcommands/failed @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" +source "$PLUGIN_AVAILABLE_PATH/apps/functions" +source "$PLUGIN_AVAILABLE_PATH/config/functions" +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +cmd-logs-failed() { + declare desc="shows the last failed deploy logs" + declare cmd="logs:failed" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1 + declare APP="$1" + local DOKKU_SCHEDULER + + if [[ -z "$APP" ]]; then + local INSTALLED_APPS=$(dokku_apps) + for app in $INSTALLED_APPS; do + fn-logs-failed-single "$app" | tee || true + done + else + fn-logs-failed-single "$APP" + fi +} + +fn-logs-failed-single() { + declare desc="shows the last failed deploy logs for an app" + declare APP="$1" + local DOKKU_SCHEDULER + + dokku_log_info2_quiet "${APP} failed deploy logs" + DOKKU_SCHEDULER=$(config_get "$APP" DOKKU_SCHEDULER || echo "docker-local") + plugn trigger scheduler-logs-failed "$DOKKU_SCHEDULER" "$APP" +} + +cmd-logs-failed "$@" diff --git a/plugins/scheduler-docker-local/install b/plugins/scheduler-docker-local/install new file mode 100755 index 000000000..a6159c0b5 --- /dev/null +++ b/plugins/scheduler-docker-local/install @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +scheduler-docker-local-install() { + declare desc="scheduler-docker-local install plugin trigger" + declare trigger="scheduler-docker-local-install" + + mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local" +} + +scheduler-docker-local-install "$@" diff --git a/plugins/scheduler-docker-local/post-create b/plugins/scheduler-docker-local/post-create new file mode 100755 index 000000000..9c9f98c29 --- /dev/null +++ b/plugins/scheduler-docker-local/post-create @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +scheduler-docker-local-post-create() { + declare desc="scheduler-docker-local post-create plugin trigger" + declare trigger="scheduler-docker-local-post-create" + declare APP="$1" + + mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/${APP}" + chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/${APP}" +} + +scheduler-docker-local-post-create "$@" diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index e4dd5bb78..c4a463ae6 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -13,6 +13,8 @@ scheduler-docker-local-scheduler-deploy() { return fi + rm -f "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers" + local DOKKU_DOCKER_STOP_TIMEOUT DOKKU_HEROKUISH DOKKU_NETWORK_BIND_ALL IMAGE DOKKU_HEROKUISH=false IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG") @@ -96,14 +98,16 @@ scheduler-docker-local-scheduler-deploy() { kill_new() { declare desc="wrapper function to kill newly started app container" - local cid="$1" - docker inspect "$cid" &> /dev/null && docker stop "$cid" > /dev/null && docker kill "$cid" &> /dev/null + declare CID="$1" PROC_TYPE="$2" CONTAINER_INDEX="$3" + docker inspect "$CID" &> /dev/null && docker stop "$CID" > /dev/null && docker kill "$CID" &> /dev/null + mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP" + echo "${CID} ${PROC_TYPE}.${CONTAINER_INDEX}" >> "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers" trap - INT TERM EXIT kill -9 $$ } # run checks first, then post-deploy hooks, which switches proxy traffic - trap 'kill_new $cid' INT TERM EXIT + trap 'kill_new $cid $PROC_TYPE $CONTAINER_INDEX' INT TERM EXIT if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "false" ]]; then dokku_log_info1 "Attempting pre-flight checks" plugn trigger check-deploy "$APP" "$cid" "$PROC_TYPE" "$port" "$ipaddr" diff --git a/plugins/scheduler-docker-local/scheduler-logs-failed b/plugins/scheduler-docker-local/scheduler-logs-failed new file mode 100755 index 000000000..1c75705f0 --- /dev/null +++ b/plugins/scheduler-docker-local/scheduler-logs-failed @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" + +scheduler-docker-local-scheduler-logs-failed() { + declare desc="scheduler-docker-local scheduler-logs-failed plugin trigger" + declare trigger="scheduler-docker-local scheduler-logs-failed" + declare DOKKU_SCHEDULER="$1" APP="$2" + local FAILED_CONTAINERS_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers" + local DEAD_CONTAINERS=() DOKKU_LOGS_CMD=() RUNNING_CONTAINERS=() + local CID LINE PREFIX + shift 2 + + if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then + return + fi + + if [[ ! -f "$FAILED_CONTAINERS_FILE" ]]; then + dokku_log_warn "No failed containers found" + return + fi + + while read -r LINE || [[ -n "$LINE" ]]; do + CID="$(echo "$LINE" | cut -d ' ' -f1)" + PREFIX="$(echo "$LINE" | cut -d ' ' -f2)" + if docker inspect "${CID}" > /dev/null 2>&1; then + RUNNING_CONTAINERS+=("$CID") + else + DEAD_CONTAINERS+=("$CID") + fi + done < "$FAILED_CONTAINERS_FILE" + + for CID in "${DEAD_CONTAINERS[@]}"; do + dokku_log_warn "App container $CID no longer running" + sed -i "/${CID}/d" "$FAILED_CONTAINERS_FILE" + done + + if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then + dokku_log_warn "No failed containers found" + return + fi + + ((MAX_INDEX=${#RUNNING_CONTAINERS[*]} - 1)) || true + for i in ${!RUNNING_CONTAINERS[*]}; do + local CID="${RUNNING_CONTAINERS[i]}" + DOKKU_LOGS_CMD+="(docker logs $DOKKU_LOGS_ARGS $CID 2>&1)" + if [[ $i != "$MAX_INDEX" ]]; then + local DOKKU_LOGS_CMD+="& " + else + local DOKKU_LOGS_CMD+="; " + fi + done + bash -c "($DOKKU_LOGS_CMD)" +} + +scheduler-docker-local-scheduler-logs-failed "$@"