From 33f872183e61c88f389bc26aff2feae03f5de64b Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Fri, 17 Apr 2015 15:23:59 -0700 Subject: [PATCH] initial pass at container-level scaling --- dokku | 106 +++++++++++++++++++------------ plugins/checks/check-deploy | 26 ++++---- plugins/common/functions | 8 +++ plugins/nginx-vhosts/commands | 23 +++++-- plugins/nginx-vhosts/post-deploy | 4 +- plugins/ps/commands | 21 +++++- plugins/ps/functions | 45 +++++++++++++ plugins/ps/pre-deploy | 8 +++ 8 files changed, 180 insertions(+), 61 deletions(-) create mode 100755 plugins/ps/functions create mode 100755 plugins/ps/pre-deploy diff --git a/dokku b/dokku index c48397b98..e3ac8da70 100755 --- a/dokku +++ b/dokku @@ -66,53 +66,78 @@ case "$1" in APP="$2"; IMAGE="dokku/$APP" pluginhook pre-deploy $APP - if [[ -f "$DOKKU_ROOT/$APP/CONTAINER" ]]; then - oldid=$(< "$DOKKU_ROOT/$APP/CONTAINER") - fi - - # start the app - DOCKER_ARGS=$(: | pluginhook docker-args $APP deploy) - DOCKER_ARGS+=$(: | pluginhook docker-args-deploy $APP) - BIND_EXTERNAL=$(pluginhook bind-external-ip $APP) - is_image_buildstep_based "$IMAGE" && DOKKU_BUILDSTEP=true - [[ -n "$DOKKU_BUILDSTEP" ]] && START_CMD="/start web" - [[ -z "$DOKKU_BUILDSTEP" ]] && DOKKU_DOCKERFILE_PORT=$(dokku config:get $APP DOKKU_DOCKERFILE_PORT || true) + DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" - if [[ "$BIND_EXTERNAL" = "false" ]];then - port=${DOKKU_DOCKERFILE_PORT:=5000} - id=$(docker run -d -e PORT=$port $DOCKER_ARGS $IMAGE $START_CMD) - ipaddr=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $id) - else - id=$(docker run -d -p 5000 -e PORT=5000 $DOCKER_ARGS $IMAGE $START_CMD) - port=$(docker port $id 5000 | sed 's/[0-9.]*://') - ipaddr=127.0.0.1 - fi + while read line || [ -n "$line" ] + do + PROC_NAME=${line%%=*} + PROC_COUNT=${line#*=} + CONTAINER_NUM=1 - # if we can't post-deploy successfully, kill new container - kill_new() { - docker inspect $id &> /dev/null && docker kill $id > /dev/null - trap - INT TERM EXIT - kill -9 $$ - } + while [[ $CONTAINER_NUM -le $PROC_COUNT ]];do + DOKKU_CONTAINER_ID_FILE="$DOKKU_ROOT/$APP/CONTAINER.$PROC_NAME.$CONTAINER_NUM" + DOKKU_IP_FILE="$DOKKU_ROOT/$APP/IP.$PROC_NAME.$CONTAINER_NUM" + DOKKU_PORT_FILE="$DOKKU_ROOT/$APP/PORT.$PROC_NAME.$CONTAINER_NUM" - # run checks first, then post-deploy hooks, which switches Nginx traffic - trap kill_new INT TERM EXIT - dokku_log_info1 "Running pre-flight checks" - pluginhook check-deploy $APP $port $ipaddr $id - trap - INT TERM EXIT + # try old form even in case we're migrating dokku versions + if [[ -f "$DOKKU_CONTAINER_ID_FILE" ]]; then + oldids+=" " + oldids+=$(< "$DOKKU_CONTAINER_ID_FILE") + oldids+=" " + elif [[ -f "$DOKKU_ROOT/$APP/CONTAINER" ]]; then + oldids+=$(< "$DOKKU_ROOT/$APP/CONTAINER") + fi - # now using the new container - echo $id > "$DOKKU_ROOT/$APP/CONTAINER" - echo $ipaddr > "$DOKKU_ROOT/$APP/IP" - echo $port > "$DOKKU_ROOT/$APP/PORT" - echo "http://$(< "$DOKKU_ROOT/HOSTNAME"):$port" > "$DOKKU_ROOT/$APP/URL" + # start the app + DOCKER_ARGS=$(: | pluginhook docker-args $APP deploy) + DOCKER_ARGS+=$(: | pluginhook docker-args-deploy $APP) + BIND_EXTERNAL=$(pluginhook bind-external-ip $APP) + + [[ -n "$DOKKU_BUILDSTEP" ]] && START_CMD="/start $PROC_NAME" + [[ -z "$DOKKU_BUILDSTEP" ]] && DOKKU_DOCKERFILE_PORT=$(dokku config:get $APP DOKKU_DOCKERFILE_PORT || true) + + + if [[ "$BIND_EXTERNAL" = "false" ]];then + port=${DOKKU_DOCKERFILE_PORT:=5000} + id=$(docker run -d -e PORT=$port $DOCKER_ARGS $IMAGE $START_CMD) + ipaddr=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $id) + else + id=$(docker run -d -p 5000 -e PORT=5000 $DOCKER_ARGS $IMAGE $START_CMD) + port=$(docker port $id 5000 | sed 's/[0-9.]*://') + ipaddr=127.0.0.1 + fi + + # if we can't post-deploy successfully, kill new container + kill_new() { + docker inspect $id &> /dev/null && docker kill $id > /dev/null + trap - INT TERM EXIT + kill -9 $$ + } + + # run checks first, then post-deploy hooks, which switches Nginx traffic + trap kill_new INT TERM EXIT + dokku_log_info1 "Running pre-flight checks" + pluginhook check-deploy $APP $port $ipaddr $id + trap - INT TERM EXIT + + # now using the new container + echo $id > "$DOKKU_CONTAINER_ID_FILE" + echo $ipaddr > "$DOKKU_IP_FILE" + echo $port > "$DOKKU_PORT_FILE" + echo "http://$(< "$DOKKU_ROOT/HOSTNAME"):$port" > "$DOKKU_ROOT/$APP/URL" + + # cleanup pre-migration files + rm -f $DOKKU_ROOT/$APP/CONTAINER $DOKKU_ROOT/$APP/IP $DOKKU_ROOT/$APP/PORT + CONTAINER_NUM=$(( CONTAINER_NUM + 1 )) + done + done < "$DOKKU_SCALE_FILE" dokku_log_info1 "Running post-deploy" - pluginhook post-deploy $APP $port $ipaddr + pluginhook post-deploy $APP $port $ipaddr # kill the old container - if [[ -n "$oldid" ]]; then + if [[ -n "$oldids" ]]; then # Let the old container finish processing requests, before terminating it WAIT="${DOKKU_WAIT_TO_RETIRE:-60}" dokku_log_info1 "Shutting down old container in $WAIT seconds" @@ -120,13 +145,14 @@ case "$1" in exec >/dev/null 2>/dev/null \"$EXPECTED\"" [[ $DOKKU_TRACE ]] && dokku_log_verbose "$ curl $CURL_ARGS" - + # Capture HTTP response or CURL error message if OUTPUT=$(curl -# $CURL_ARGS 2>&1) ; then # OUTPUT contains the HTTP response @@ -178,7 +178,7 @@ do FAILEDCHECKS=$(( FAILEDCHECKS + 1 )) fi done - + if [ $FAILEDCHECKS -gt 0 ]; then dokku_log_warn "Check attempt $ATTEMPT/$ATTEMPTS failed." SUCCESS=0 diff --git a/plugins/common/functions b/plugins/common/functions index 8db3af6dd..a48b5c499 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -135,3 +135,11 @@ parse_args() { done return 0 } + +copy_from_image() { + local APP="$1"; local IMAGE="dokku/$APP"; local SRC_FILE="$2"; local DST_DIR="$3"; verify_app_name $APP + + CID=$(docker run -d $IMAGE bash) + docker cp "$CID:$SRC_FILE" "$DST_DIR" + docker rm -f "$CID" +} diff --git a/plugins/nginx-vhosts/commands b/plugins/nginx-vhosts/commands index 19b48df10..bd52c7c58 100755 --- a/plugins/nginx-vhosts/commands +++ b/plugins/nginx-vhosts/commands @@ -34,11 +34,18 @@ case "$1" in WILDCARD_SSL="$DOKKU_ROOT/tls" SSL="$DOKKU_ROOT/$APP/tls" - if [[ -z "$DOKKU_APP_LISTEN_PORT" ]] && [[ -f "$DOKKU_ROOT/$APP/PORT" ]]; then - DOKKU_APP_LISTEN_PORT=$(< "$DOKKU_ROOT/$APP/PORT") - fi - if [[ -z "$DOKKU_APP_LISTEN_IP" ]] && [[ -f "$DOKKU_ROOT/$APP/IP" ]]; then - DOKKU_APP_LISTEN_IP=$(< "$DOKKU_ROOT/$APP/IP") + if [[ -z "$DOKKU_APP_LISTEN_PORT" ]] && [[ -z "$DOKKU_APP_LISTEN_IP" ]]; then + shopt -s nullglob + for DOKKU_APP_IP_FILE in $DOKKU_ROOT/$APP/IP.*;do + DOKKU_APP_PORT_FILE=$(echo $DOKKU_APP_IP_FILE | sed -e "s:IP:PORT:g") + DOKKU_APP_LISTENER_IP=$(< $DOKKU_APP_IP_FILE) + DOKKU_APP_LISTENER_PORT=$(< $DOKKU_APP_PORT_FILE) + + DOKKU_APP_LISTENERS+=" " + DOKKU_APP_LISTENERS+="$DOKKU_APP_LISTENER_IP:$DOKKU_APP_LISTENER_PORT" + DOKKU_APP_LISTENERS+=" " + done + shopt -u nullglob fi [[ -f "$DOKKU_ROOT/ENV" ]] && source $DOKKU_ROOT/ENV @@ -100,6 +107,12 @@ EOF if [[ -n "$DOKKU_APP_LISTEN_PORT" ]] && [[ -n "$DOKKU_APP_LISTEN_IP" ]]; then echo "upstream $APP { server $DOKKU_APP_LISTEN_IP:$DOKKU_APP_LISTEN_PORT; }" >> $NGINX_CONF + elif [[ -n "$DOKKU_APP_LISTENERS" ]]; then + echo "upstream $APP { " >> $NGINX_CONF + for listener in $DOKKU_APP_LISTENERS;do + echo " server $listener;" >> $NGINX_CONF + done + echo "}" >> $NGINX_CONF fi dokku_log_info1 "Creating $SCHEME nginx.conf" diff --git a/plugins/nginx-vhosts/post-deploy b/plugins/nginx-vhosts/post-deploy index c803c980b..3b1940dc3 100755 --- a/plugins/nginx-vhosts/post-deploy +++ b/plugins/nginx-vhosts/post-deploy @@ -2,7 +2,7 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x source "$(dirname $0)/../common/functions" -APP="$1"; PORT="$2"; IP="$3" +APP="$1" NO_VHOST=$(dokku config:get $APP NO_VHOST || true) if [[ -n "$NO_VHOST" ]]; then @@ -11,4 +11,4 @@ elif [[ ! -f "$DOKKU_ROOT/$APP/VHOST" ]]; then dokku domains:setup $APP fi -dokku nginx:build-config $APP $PORT $IP +dokku nginx:build-config $APP diff --git a/plugins/ps/commands b/plugins/ps/commands index 4fd227b76..be707dc10 100755 --- a/plugins/ps/commands +++ b/plugins/ps/commands @@ -1,12 +1,13 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x source "$(dirname $0)/../common/functions" +source "$(dirname $0)/functions" release_and_deploy() { source "$(dirname $0)/../common/functions" local APP="$1"; local IMAGE="dokku/$APP" - if [[ -f "$DOKKU_ROOT/$APP/CONTAINER" ]]; then + if ls $DOKKU_ROOT/$APP/CONTAINER.* &> /dev/null; then if is_image_buildstep_based "$IMAGE"; then IMAGE_SOURCE_TYPE="buildstep" else @@ -97,9 +98,27 @@ case "$1" in done ;; + ps:scale) + [[ -z $2 ]] && echo "Please specify an app to run the command on" && exit 1 + verify_app_name "$2" + + APP="$2"; DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" + shift 2 + + generate_scale_file "$APP" + if [[ -z "$@" ]];then + dokku_log_info1 "Scaling for $APP" + dokku_log_info2 "$(< $DOKKU_SCALE_FILE)" + else + set_scale "$APP" "$@" + fi + release_and_deploy "$APP" + ;; + help | ps:help) cat && cat< List processes running in app container(s) + ps:scale = [=] Set how many processes of a given process to run ps:start Start app container(s) ps:stop Stop app container(s) ps:rebuild Rebuild an app diff --git a/plugins/ps/functions b/plugins/ps/functions new file mode 100755 index 000000000..9664e875d --- /dev/null +++ b/plugins/ps/functions @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$(dirname $0)/../common/functions" + +generate_scale_file() { + local APP="$1"; local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" + if [[ ! -f $DOKKU_SCALE_FILE ]]; then + copy_from_image "$APP" "/app/DOKKU_SCALE" "$DOKKU_ROOT/$APP" || dokku_log_info1_quiet "DOKKU_SCALE not found in app image" + if [[ ! -f $DOKKU_SCALE_FILE ]]; then + local TMP_WORK_DIR=$(mktemp -d DOKKU_SCALE.XXXXX) + trap 'rm -rf $TMP_WORK_DIR' EXIT + + copy_from_image "$APP" /app/Procfile $TMP_WORK_DIR || true + if [[ -f $TMP_WORK_DIR/Procfile ]];then + local PROCFILE="$TMP_WORK_DIR/Procfile" + while read line || [ -n "$line" ] + do + local PROC_NAME=${line%%:*} + echo "$PROC_NAME=1" >> $DOKKU_SCALE_FILE + done < "$PROCFILE" + else + echo "default=1" >> $DOKKU_SCALE_FILE + fi + + dokku_log_info1_quiet "New DOKKU_SCALE file generated" + dokku_log_info1_quiet "$(< $DOKKU_SCALE_FILE)" + fi + fi +} + +set_scale() { + local APP="$1"; local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" + shift 1 + local SCALE_SETTINGS=("$@") + for procscale in "${SCALE_SETTINGS[@]}"; do + PROC_NAME=${procscale%%=*} + PROC_COUNT=${procscale#*=} + dokku_log_info1_quiet "Scaling $APP:$PROC_NAME to $PROC_COUNT" + if (egrep -q ^${PROC_NAME}= $DOKKU_SCALE_FILE > /dev/null 2>&1);then + sed --in-place "s:^${PROC_NAME}=.*:$PROC_NAME=$PROC_COUNT:g" $DOKKU_SCALE_FILE + else + echo "$PROC_NAME=$PROC_COUNT" >> $DOKKU_SCALE_FILE + fi + done +} diff --git a/plugins/ps/pre-deploy b/plugins/ps/pre-deploy new file mode 100755 index 000000000..bc4f77c94 --- /dev/null +++ b/plugins/ps/pre-deploy @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +source "$(dirname $0)/../common/functions" +source "$(dirname $0)/functions" + +APP="$1" + +generate_scale_file $APP