Merge pull request #5475 from dokku/5207-nginx-core-post-extract

Refactor nginx.conf.sigil to be extracted from a specified path in source code instead of the built image

docs/appendices/0.29.0-migration-guide.md

@@ -4,6 +4,7 @@
 
 - The output of `run:detached` now uses the container name - eg. `node-js-app.run.1` - vs the container id.
 - The ID of `cron` tasks is now base36-encoded instead of base64-encoded.
+- The `nginx.conf.sigil` is now extracted when source code is extracted for a build and not from the built image. Users can specify alternative paths via the `nginx-conf-sigil-path` property of the `nginx` plugin. See the [nginx documentation](/docs/networking/proxies/nginx.md#customizing-the-nginx-configuration) for more information on how to configure the `nginx.conf.sigil` path for your application.
 
 ## Removals
 

docs/networking/proxies/nginx.md

@@ -287,15 +287,26 @@ dokku nginx:validate-config node-js-app --clean
 
 > New as of 0.5.0
 
-Dokku uses a templating library by the name of [sigil](https://github.com/gliderlabs/sigil) to generate nginx configuration for each app. You may also provide a custom template for your application as follows:
+Dokku uses a templating library by the name of [sigil](https://github.com/gliderlabs/sigil) to generate nginx configuration for each app. This may be overriden by committing the [default configuration template](https://github.com/dokku/dokku/blob/master/plugins/nginx-vhosts/templates/nginx.conf.sigil) to a file named `nginx.conf.sigil` in the root of the app repository.
 
-- Copy the [default configuration template](https://github.com/dokku/dokku/blob/master/plugins/nginx-vhosts/templates/nginx.conf.sigil) to a file named `nginx.conf.sigil` and either:
-  - If using a buildpack application, you __must__ check it into the root of your app repo.
-  - If using a dockerfile or docker image for deploys, either:
-    - If `WORKDIR` is specified, add the file to the `WORKDIR` specified in the last Dockerfile stage  (example: `WORKDIR /app` and `ADD nginx.conf.sigil /app`).
-    - If no `WORKDIR` is specified, add the file to the root (`/`) of the docker image (example: `ADD nginx.conf.sigil /`).
+By default, the `nginx.conf.sigil` path is pulled from the root of the app source code repository. To change the path from which the `nginx.conf.sigil` is extracted, the `nginx-conf-sigil-path` property can be modified via `nginx:set`.
 
-> When using a custom `nginx.conf.sigil` file, depending upon your application configuration, you _may_ be exposing the file externally. As this file is extracted before the container is run, you can, safely delete it in a custom `entrypoint.sh` configured in a Dockerfile `ENTRYPOINT`.
+```shell
+dokku nginx:set node-js-app nginx-conf-sigil-path dokku/nginx.conf.sigil
+```
+
+This property can also be changed globally, which will take into effect if there is no value at the app level.
+
+```shell
+dokku nginx:set --global nginx-conf-sigil-path dokku/nginx.conf.sigil
+```
+
+In either case, the value can be reset by specifying an empty value.
+
+```shell
+dokku nginx:set node-js-app nginx-conf-sigil-path
+dokku nginx:set --global nginx-conf-sigil-path
+```
 
 > The [default template](https://github.com/dokku/dokku/blob/master/plugins/nginx-vhosts/templates/nginx.conf.sigil) may change with new releases of Dokku. Please refer to the appropriate template file version for your Dokku version, and make sure to look out for changes when you upgrade.
 

plugins/nginx-vhosts/command-functions

@@ -50,6 +50,9 @@ cmd-nginx-report-single() {
     "--nginx-hsts-include-subdomains: $(fn-plugin-property-get-default "nginx" "$APP" "hsts-include-subdomains" "true")"
     "--nginx-hsts-max-age: $(fn-plugin-property-get-default "nginx" "$APP" "hsts-max-age" "15724800")"
     "--nginx-hsts-preload: $(fn-plugin-property-get-default "nginx" "$APP" "hsts-preload" "false")"
+    "--nginx-computed-nginx-conf-sigil-path: $(fn-nginx-computed-nginx-conf-sigil-path "$APP")"
+    "--nginx-global-nginx-conf-sigil-path: $(fn-nginx-global-nginx-conf-sigil-path)"
+    "--nginx-nginx-conf-sigil-path: $(fn-nginx-nginx-conf-sigil-path "$APP")"
     "--nginx-proxy-buffer-size: $(fn-nginx-proxy-buffer-size "$APP")"
     "--nginx-proxy-buffering: $(fn-nginx-proxy-buffering "$APP")"
     "--nginx-proxy-buffers: $(fn-nginx-proxy-buffers "$APP")"

plugins/nginx-vhosts/core-post-deploy

@@ -18,6 +18,13 @@ trigger-nginx-vhosts-core-post-deploy() {
   if [[ "$HAS_NETWORK_CONFIG" == "true" ]]; then
     plugn trigger domains-setup "$APP"
     nginx_build_config "$APP"
+    if [[ -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID" ]]; then
+      mv "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID" "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil"
+    fi
+    if [[ -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID.missing" ]]; then
+      rm -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID.missing"
+      rm -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil"
+    fi
   fi
 }
 

plugins/nginx-vhosts/core-post-extract

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+source "$PLUGIN_AVAILABLE_PATH/nginx-vhosts/internal-functions"
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+
+trigger-nginx-vhosts-core-post-extract() {
+  declare desc="nginx-vhosts post-extract plugin trigger"
+  declare trigger="post-extract"
+  declare APP="$1" SOURCECODE_WORK_DIR="$2"
+  local CONF_SIGIL_PATH="$(fn-nginx-computed-nginx-conf-sigil-path "$APP")"
+
+  pushd "$TMP_WORK_DIR" >/dev/null
+  mkdir -p "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP"
+
+  if [[ -z "$CONF_SIGIL_PATH" ]]; then
+    touch "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID.missing"
+    return
+  fi
+
+  if [[ ! -f "$CONF_SIGIL_PATH" ]]; then
+    touch "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID.missing"
+    return
+  fi
+
+  rm -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil."*
+  cp -f "$CONF_SIGIL_PATH" "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID"
+  popd &>/dev/null || pushd "/tmp" >/dev/null
+}
+
+trigger-nginx-vhosts-core-post-extract "$@"

plugins/nginx-vhosts/functions

@@ -190,7 +190,15 @@ get_custom_nginx_template() {
     return
   fi
 
-  copy_from_image "$IMAGE" "$NGINX_TEMPLATE_NAME" "$DESTINATION_FILE" 2>/dev/null || true
+  if [[ -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID.missing" ]]; then
+    return
+  fi
+
+  if [[ -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID" ]]; then
+    cp "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil.$DOKKU_PID" "$DESTINATION_FILE"
+  elif [[ -f "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil" ]]; then
+    cp "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil" "$DESTINATION_FILE"
+  fi
 }
 
 is_tls13_available() {

plugins/nginx-vhosts/install

@@ -6,6 +6,48 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
 source "$PLUGIN_AVAILABLE_PATH/config/functions"
 source "$PLUGIN_AVAILABLE_PATH/nginx-vhosts/internal-functions"
 
+fn-nginx-vhosts-migrate-env-vars() {
+  # @TODO: Remove this after a few versions
+  for app in $(dokku_apps "false"); do
+    nginx_port="$(config_get "$app" DOKKU_NGINX_PORT || true)"
+    nginx_ssl_port="$(config_get "$app" DOKKU_NGINX_SSL_PORT || true)"
+    if [[ -n "$nginx_port" ]] || [[ -n "$nginx_ssl_port" ]]; then
+      dokku_log_info1 "Migrating DOKKU_NGINX env variables. The following variables will be migrated"
+      dokku_log_info2 "DOKKU_NGINX_PORT -> DOKKU_PROXY_PORT"
+      dokku_log_info2 "DOKKU_NGINX_SSL_PORT -> DOKKU_PROXY_SSL_PORT"
+    fi
+    if [[ -n "$nginx_port" ]]; then
+      dokku_log_info1 "Migrating DOKKU_NGINX_PORT to DOKKU_PROXY_PORT for $app"
+      DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$app" DOKKU_PROXY_PORT="$nginx_port"
+      DOKKU_QUIET_OUTPUT=1 config_unset --no-restart "$app" DOKKU_NGINX_PORT
+    fi
+    if [[ -n "$nginx_ssl_port" ]]; then
+      dokku_log_info1 "Migrating DOKKU_NGINX_SSL_PORT to DOKKU_PROXY_SSL_PORT for $app"
+      DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$app" DOKKU_PROXY_SSL_PORT="$nginx_ssl_port"
+      DOKKU_QUIET_OUTPUT=1 config_unset --no-restart "$app" DOKKU_NGINX_SSL_PORT
+    fi
+  done
+}
+
+fn-nginx-vhosts-migrate-nginx-conf-sigil() {
+  if [[ "$(fn-plugin-property-get-default "nginx" "--global" "nginx-conf-sigil-migrated" "")" == "true" ]]; then
+    return
+  fi
+
+  fn-plugin-property-write "nginx" "--global" "nginx-conf-sigil-migrated" "true"
+
+  for APP in $(dokku_apps "false"); do
+    local DISABLE_CUSTOM_CONFIG="$(fn-plugin-property-get-default "nginx" "$APP" "disable-custom-config" "false")"
+    if [[ "$DISABLE_CUSTOM_CONFIG" == "true" ]]; then
+      continue
+    fi
+
+    local IMAGE_TAG="$(get_running_image_tag "$APP")"
+    local IMAGE=$(get_deploying_app_image_name "$APP" "$IMAGE_TAG")
+    copy_from_image "$IMAGE" "nginx.conf.sigil" "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP/nginx.conf.sigil" 2>/dev/null || true
+  done
+}
+
 trigger-nginx-vhosts-install() {
   declare desc="nginx-vhosts install trigger"
   declare trigger="install"
@@ -85,26 +127,8 @@ trigger-nginx-vhosts-install() {
   # patch broken nginx 1.8.0 logrotate
   [[ -f /etc/logrotate.d/nginx ]] && sed -i -e 's/invoke-rc.d/service/g' /etc/logrotate.d/nginx
 
-  # @TODO: Remove this after a few versions
-  for app in $(dokku_apps "false"); do
-    nginx_port="$(config_get "$app" DOKKU_NGINX_PORT || true)"
-    nginx_ssl_port="$(config_get "$app" DOKKU_NGINX_SSL_PORT || true)"
-    if [[ -n "$nginx_port" ]] || [[ -n "$nginx_ssl_port" ]]; then
-      dokku_log_info1 "Migrating DOKKU_NGINX env variables. The following variables will be migrated"
-      dokku_log_info2 "DOKKU_NGINX_PORT -> DOKKU_PROXY_PORT"
-      dokku_log_info2 "DOKKU_NGINX_SSL_PORT -> DOKKU_PROXY_SSL_PORT"
-    fi
-    if [[ -n "$nginx_port" ]]; then
-      dokku_log_info1 "Migrating DOKKU_NGINX_PORT to DOKKU_PROXY_PORT for $app"
-      DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$app" DOKKU_PROXY_PORT="$nginx_port"
-      DOKKU_QUIET_OUTPUT=1 config_unset --no-restart "$app" DOKKU_NGINX_PORT
-    fi
-    if [[ -n "$nginx_ssl_port" ]]; then
-      dokku_log_info1 "Migrating DOKKU_NGINX_SSL_PORT to DOKKU_PROXY_SSL_PORT for $app"
-      DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$app" DOKKU_PROXY_SSL_PORT="$nginx_ssl_port"
-      DOKKU_QUIET_OUTPUT=1 config_unset --no-restart "$app" DOKKU_NGINX_SSL_PORT
-    fi
-  done
+  fn-nginx-vhosts-migrate-env-vars
+  fn-nginx-vhosts-migrate-nginx-conf-sigil
 
   # avoid failing runit init calls on install
   # the runit binaries are not yet available during dockerfile building

plugins/nginx-vhosts/internal-functions

@@ -24,6 +24,28 @@ fn-nginx-hsts-is-enabled() {
   echo "$hsts_is_enabled"
 }
 
+fn-nginx-computed-nginx-conf-sigil-path() {
+  declare APP="$1"
+  local conf_sigil_path="$(fn-nginx-nginx-conf-sigil-path "$APP")"
+
+  if [[ "$conf_sigil_path" == "" ]]; then
+    conf_sigil_path="$(fn-nginx-global-nginx-conf-sigil-path)"
+  fi
+
+  echo "$conf_sigil_path"
+}
+
+fn-nginx-global-nginx-conf-sigil-path() {
+  declare APP="$1"
+
+  fn-plugin-property-get-default "nginx" "--global" "nginx-conf-sigil-path" "nginx.conf.sigil"
+}
+
+fn-nginx-nginx-conf-sigil-path() {
+  declare APP="$1"
+  fn-plugin-property-get-default "nginx" "$APP" "nginx-conf-sigil-path" ""
+}
+
 fn-nginx-vhosts-manage-hsts() {
   declare APP="$1" SSL_ENABLED="$2"
   local HSTS="$(fn-nginx-hsts-is-enabled "$APP")"

plugins/nginx-vhosts/post-delete

@@ -11,6 +11,7 @@ trigger-nginx-vhosts-post-delete() {
 
   fn-plugin-property-destroy "nginx" "$APP"
   restart_nginx "$@" >/dev/null
+  rm -rf "${DOKKU_LIB_ROOT}/data/nginx-vhosts/app-$APP"
 }
 
 trigger-nginx-vhosts-post-delete "$@"

plugins/nginx-vhosts/subcommands/set

@@ -9,13 +9,13 @@ cmd-nginx-set() {
   declare cmd="nginx:set"
   [[ "$1" == "$cmd" ]] && shift 1
   declare APP="$1" KEY="$2" VALUE="$3"
-  local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "proxy-read-timeout" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl")
+  local VALID_KEYS=("access-log-format" "access-log-path" "bind-address-ipv4" "bind-address-ipv6" "client-max-body-size" "disable-custom-config" "error-log-path" "hsts" "hsts-include-subdomains" "hsts-preload" "hsts-max-age" "nginx-conf-sigil-path" "proxy-read-timeout" "proxy-buffer-size" "proxy-buffering" "proxy-buffers" "proxy-busy-buffers-size" "x-forwarded-for-value" "x-forwarded-port-value" "x-forwarded-proto-value" "x-forwarded-ssl")
   local GLOBAL_KEYS=("hsts")
 
   [[ -z "$KEY" ]] && dokku_log_fail "No key specified"
 
   if ! fn-in-array "$KEY" "${VALID_KEYS[@]}"; then
-    dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-max-body-size, disable-custom-config, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, proxy-read-timeout, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl"
+    dokku_log_fail "Invalid key specified, valid keys include: access-log-format, access-log-path, bind-address-ipv4, bind-address-ipv6, client-max-body-size, disable-custom-config, error-log-path, hsts, hsts-include-subdomains, hsts-preload, hsts-max-age, nginx-conf-sigil-path, proxy-read-timeout, proxy-buffer-size, proxy-buffering, proxy-buffers, proxy-busy-buffers-size, x-forwarded-for-value, x-forwarded-port-value, x-forwarded-proto-value, x-forwarded-ssl"
   fi
 
   if ! fn-in-array "$KEY" "${GLOBAL_KEYS[@]}"; then

tests/unit/nginx-vhosts_11.bats

@@ -149,3 +149,66 @@ teardown() {
   echo "status: $status"
   assert_output_contains "proxy_busy_buffers_size 10k;" 0
 }
+
+@test "(nginx-vhosts) nginx:set nginx.conf.sigil" {
+  local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX")
+  trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM
+  export CUSTOM_TMP="$TMP"
+
+  run deploy_app python
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Overriding default nginx.conf with detected nginx.conf.sigil" 0
+
+  pushd $TMP
+  custom_nginx_template "$TEST_APP" "$CUSTOM_TMP"
+
+  run /bin/bash -c "git add nginx.conf.sigil"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "git commit -m 'Add custom nginx.conf.sigil'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "git push target master:master"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Overriding default nginx.conf with detected nginx.conf.sigil"
+
+  mv "$CUSTOM_TMP/nginx.conf.sigil" "$CUSTOM_TMP/nginx.conf.sigil-2"
+
+  run /bin/bash -c "git add nginx.conf.sigil nginx.conf.sigil-2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "git commit -m 'Moving nginx.conf.sigil'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "git push target master:master"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Overriding default nginx.conf with detected nginx.conf.sigil" 0
+
+  run /bin/bash -c "dokku nginx:set $TEST_APP nginx-conf-sigil-path nginx.conf.sigil-2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:rebuild $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Overriding default nginx.conf with detected nginx.conf.sigil"
+
+  popd
+  rm -rf "$TMP"
+}