feat: add a subcommand for retrieving failed app deploy logs

This can be useful in cases where a developer wishes to see what error messages are available for a failed deploy, but does not wish to spelunk around `docker ps -a` output.

Note that the logs are not made available once a further deploy is triggered, or after the containers have been garbage collected.

docs/deployment/logs.md

@@ -1,19 +1,21 @@
 # Log Management
 
 ```
-logs <app>           # display recent log output
+logs <app> [-h] [-t] [-n num] [-q] [-p process]  # Display recent log output
+logs:failed <app>                                # Shows the last failed deploy logs for an app
 ```
 
 ## Usage
 
-### Application Logs
-You can easily get logs of an application using the `logs` command:
+### App Logs
+
+You can easily get logs of an app using the `logs` command:
 
 ```shell
 dokku logs node-js-app
 ```
 
-## Behavioral modifiers
+#### Behavioral modifiers
 
 Dokku also supports certain command-line arguments that augment the `log` command's behavior.
 
@@ -26,8 +28,19 @@ Dokku also supports certain command-line arguments that augment the `log` comman
 
 You can use these modifiers as follows:
 
-
 ```shell
 dokku logs node-js-app -t -p web
 ```
 will show logs continually from the web process.
+
+### Failed Deploy Logs
+
+> Warning: The default `docker-local` scheduler will "store" these until the next deploy or until the old containers are garbage collected - whichever runs first. If you require the logs beyond this point in time, please ship the logs to a centralized log server.
+
+In some cases, it may be useful to retrieve the logs from a previously failed deploy.
+
+You can retrieve these logs by using the `logs:failed` command
+
+```shell
+dokku logs:failed node-js-app
+```

dokku

@@ -42,6 +42,8 @@ export PLUGIN_ENABLED_PATH=${PLUGIN_ENABLED_PATH:="$PLUGIN_PATH/enabled"}
 export PLUGIN_CORE_PATH=${PLUGIN_CORE_PATH:="$DOKKU_LIB_ROOT/core-plugins"}
 export PLUGIN_CORE_AVAILABLE_PATH=${PLUGIN_CORE_AVAILABLE_PATH:="$PLUGIN_CORE_PATH/available"}
 export PLUGIN_CORE_ENABLED_PATH=${PLUGIN_CORE_ENABLED_PATH:="$PLUGIN_CORE_PATH/enabled"}
+export DOKKU_SYSTEM_GROUP=${DOKKU_SYSTEM_GROUP:="dokku"}
+export DOKKU_SYSTEM_USER=${DOKKU_SYSTEM_USER:="dokku"}
 
 export DOKKU_API_VERSION=1
 export DOKKU_NOT_IMPLEMENTED_EXIT=10

plugins/common/property-functions

@@ -67,15 +67,6 @@ fn-plugin-property-setup() {
 
   mkdir -p "${DOKKU_LIB_ROOT}/config/${PLUGIN}"
   chmod 0755 "${DOKKU_LIB_ROOT}/config/${PLUGIN}"
-
-  if [[ -z "$DOKKU_SYSTEM_GROUP" ]]; then
-    export DOKKU_SYSTEM_GROUP=dokku
-  fi
-
-  if [[ -z "$DOKKU_SYSTEM_USER" ]]; then
-    export DOKKU_SYSTEM_USER=dokku
-  fi
-
   chown "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/config/${PLUGIN}"
 }
 
@@ -86,7 +77,7 @@ fn-plugin-property-read() {
   if [[ -z "$PLUGIN" ]] || [[ -z "$APP" ]] || [[ -z "$KEY" ]]; then
     return 1
   fi
-  
+
   if [[ ! -f "${DOKKU_LIB_ROOT:?}/config/${PLUGIN}/${APP}/${KEY}" ]]; then
     return 1
   fi

plugins/logs/commands

@@ -1,27 +1,11 @@
 #!/usr/bin/env bash
 [[ " help logs:help " == *" $1 "* ]] || exit "$DOKKU_NOT_IMPLEMENTED_EXIT"
+source "$PLUGIN_AVAILABLE_PATH/logs/internal-functions"
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
 
-usage() {
-  declare desc="logs specific usage"
-  echo "Usage: dokku logs <app>"
-  echo " display recent log output"
-  echo ""
-  echo " -n, --num NUM        # the number of lines to display"
-  echo " -p, --ps PS          # only display logs from the given process"
-  echo " -t, --tail           # continually stream logs"
-  echo " -q, --quiet          # display raw logs without colors, time and names"
-}
-
 case "$1" in
-  logs:help)
-    usage
-    ;;
-
-  help)
-    cat<<EOF
-    logs <app> [-h] [-t] [-n num] [-q] [-p process], Show the last logs for an application
-EOF
+  help | logs:help)
+    cmd-logs-help "$@"
     ;;
 
   *)

plugins/logs/internal-functions

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+
+fn-logs-usage() {
+  declare desc="logs specific usage"
+  echo "Usage: dokku logs <app>"
+  echo " display recent log output"
+  echo ""
+  echo " -n, --num NUM        # the number of lines to display"
+  echo " -p, --ps PS          # only display logs from the given process"
+  echo " -t, --tail           # continually stream logs"
+  echo " -q, --quiet          # display raw logs without colors, time and names"
+}
+
+fn-logs-help-content() {
+  declare desc="return logs plugin help content"
+  cat<<help_content
+    logs <app> [-h] [-t] [-n num] [-q] [-p process], Display recent log output
+    logs:failed <app>, Show the last logs for an application
+help_content
+}
+
+cmd-logs-help() {
+  if [[ $1 = "logs:help" ]] ; then
+    echo -e 'Usage: dokku logs[:COMMAND]'
+    echo ''
+    echo 'Output app logs'
+    echo ''
+    echo 'Additional commands:'
+    fn-logs-help-content | sort | column -c2 -t -s,
+  elif [[ $(ps -o command= $PPID) == *"--all"* ]]; then
+    fn-logs-help-content
+  else
+    cat<<help_desc
+    logs, Output app logs
+help_desc
+  fi
+}

plugins/logs/subcommands/default

@@ -1,24 +1,22 @@
 #!/usr/bin/env bash
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
 source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+source "$PLUGIN_AVAILABLE_PATH/logs/internal-functions"
 
 logs_default_cmd() {
-  declare desc="shows the last logs for an app via command line"
+  declare desc="display recent log output"
   local cmd="logs"
   [[ -z $2 ]] && dokku_log_fail "Please specify an app to run the command on"
   local APP="$2"; verify_app_name "$2"
   local APP_ROOT="$DOKKU_ROOT/$APP"
   local COLORS=(36 33 32 35 31)
 
-  if ! (is_deployed "$APP"); then
-    echo "Application's container not found"
-    exit 1
-  fi
+  ! (is_deployed "$APP") && dokku_log_fail "App $APP has not been deployed"
 
   shift 2;
   local TEMP=$(getopt -o htqn:p: --long help,tail,quiet,num:,ps: -n 'dokku logs' -- "$@")
   local EXIT_CODE="$?"
-  if [[ "$EXIT_CODE" != 0 ]]; then usage >&2 ; exit 1 ; fi
+  if [[ "$EXIT_CODE" != 0 ]]; then fn-logs-usage >&2 ; exit 1 ; fi
   eval set -- "$TEMP"
 
   local DOKKU_LOGS_LINE_NUMBERS="100"
@@ -29,7 +27,7 @@ logs_default_cmd() {
           -p|--ps) local DOKKU_LOGS_ONLY_PROCESS="$2"; shift 2 ;;
           -q|--quiet) local DOKKU_LOGS_NO_PRETTY_PRINT="true"; shift ;;
           --) shift; break ;;
-          *) echo "Internal error"; exit 1;;
+          *) dokku_log_fail "Internal error";;
       esac
   done
 

plugins/logs/subcommands/failed

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+source "$PLUGIN_AVAILABLE_PATH/config/functions"
+
+cmd-logs-failed() {
+  declare desc="shows the last failed deploy logs for an app"
+  declare cmd="logs:failed" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1
+  declare APP="$1"
+  local DOKKU_SCHEDULER
+  [[ -z "$APP" ]] && dokku_log_fail "Please specify an app to run the command on"
+
+  DOKKU_SCHEDULER=$(config_get "$APP" DOKKU_SCHEDULER || echo "docker-local")
+  plugn trigger scheduler-failed "$DOKKU_SCHEDULER" "$APP"
+}
+
+cmd-logs-failed "$@"

plugins/scheduler-docker-local/install

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+
+scheduler-docker-local-install() {
+  declare desc="scheduler-docker-local install plugin trigger"
+  declare trigger="scheduler-docker-local-install"
+
+  mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local"
+  chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local"
+}
+
+scheduler-docker-local-install "$@"

plugins/scheduler-docker-local/post-create

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+
+scheduler-docker-local-post-create() {
+  declare desc="scheduler-docker-local post-create plugin trigger"
+  declare trigger="scheduler-docker-local-post-create"
+  declare APP="$1"
+
+  mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/${APP}"
+  chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/${APP}"
+}
+
+scheduler-docker-local-post-create "$@"

plugins/scheduler-docker-local/scheduler-deploy

@@ -13,6 +13,8 @@ scheduler-docker-local-scheduler-deploy() {
     return
   fi
 
+  rm -f "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers"
+
   local DOKKU_DOCKER_STOP_TIMEOUT DOKKU_HEROKUISH DOKKU_NETWORK_BIND_ALL IMAGE
   DOKKU_HEROKUISH=false
   IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")
@@ -96,14 +98,16 @@ scheduler-docker-local-scheduler-deploy() {
 
       kill_new() {
         declare desc="wrapper function to kill newly started app container"
-        local cid="$1"
-        docker inspect "$cid" &> /dev/null && docker stop "$cid" > /dev/null && docker kill "$cid" &> /dev/null
+        declare CID="$1" PROC_TYPE="$2" CONTAINER_INDEX="$3"
+        docker inspect "$CID" &> /dev/null && docker stop "$CID" > /dev/null && docker kill "$CID" &> /dev/null
+        mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP"
+        echo "${CID} ${PROC_TYPE}.${CONTAINER_INDEX}" >> "${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers"
         trap - INT TERM EXIT
         kill -9 $$
       }
 
       # run checks first, then post-deploy hooks, which switches proxy traffic
-      trap 'kill_new $cid' INT TERM EXIT
+      trap 'kill_new $cid $PROC_TYPE $CONTAINER_INDEX' INT TERM EXIT
       if [[ "$(is_app_proctype_checks_disabled "$APP" "$PROC_TYPE")" == "false" ]]; then
         dokku_log_info1 "Attempting pre-flight checks"
         plugn trigger check-deploy "$APP" "$cid" "$PROC_TYPE" "$port" "$ipaddr"

plugins/scheduler-docker-local/scheduler-failed

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+
+scheduler-docker-local-scheduler-failed() {
+  declare desc="scheduler-docker-local scheduler-failed plugin trigger"
+  declare trigger="scheduler-docker-local scheduler-failed"
+  declare DOKKU_SCHEDULER="$1" APP="$2"
+  local FAILED_CONTAINERS_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/$APP/failed-containers"
+  local DEAD_CONTAINERS=() DOKKU_LOGS_CMD=() RUNNING_CONTAINERS=()
+  local CID LINE PREFIX
+  shift 2
+
+  if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then
+    return
+  fi
+
+  if [[ ! -f "$FAILED_CONTAINERS_FILE" ]]; then
+    dokku_log_warn "No failed containers found"
+    return
+  fi
+
+  while read -r LINE || [[ -n "$LINE" ]]; do
+    CID="$(echo "$LINE" | cut -d ' ' -f1)"
+    PREFIX="$(echo "$LINE" | cut -d ' ' -f2)"
+    if docker inspect "${CID}" > /dev/null 2>&1; then
+      RUNNING_CONTAINERS+=("$CID")
+    else
+      DEAD_CONTAINERS+=("$CID")
+    fi
+  done < "$FAILED_CONTAINERS_FILE"
+
+  for CID in "${DEAD_CONTAINERS[@]}"; do
+    dokku_log_warn "App container $CID no longer running"
+    sed -i "/${CID}/d" "$FAILED_CONTAINERS_FILE"
+  done
+
+  if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
+    dokku_log_warn "No failed containers found"
+    return
+  fi
+
+  ((MAX_INDEX=${#RUNNING_CONTAINERS[*]} - 1)) || true
+  for i in ${!RUNNING_CONTAINERS[*]}; do
+    local CID="${RUNNING_CONTAINERS[i]}"
+    DOKKU_LOGS_CMD+="(docker logs $DOKKU_LOGS_ARGS $CID 2>&1)"
+    if [[ $i != "$MAX_INDEX" ]]; then
+      local DOKKU_LOGS_CMD+="& "
+    else
+      local DOKKU_LOGS_CMD+="; "
+    fi
+  done
+  bash -c "($DOKKU_LOGS_CMD)"
+}
+
+scheduler-docker-local-scheduler-failed "$@"