github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: correctly handle injecting environment variables to deployment tasks

Browse Source 
Also test that release tasks are invoked the correct environment.

Closes #3817
This commit is contained in:
Jose Diaz-Gonzalez
2020-08-31 15:01:14 -04:00
parent 7e7931f1ce
commit 84367e1c16
26 changed files with 449 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
plugins/app-json/appjson.go
View File

@@ -3,12 +3,14 @@ package appjson
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"strings"
"github.com/dokku/dokku/plugins/common"
shellquote "github.com/kballard/go-shellquote"
)
type AppJson struct {
@@ -133,7 +135,7 @@ func executeScript(appName string, imageTag string, phase string) error {
return nil
}
common.LogInfo1(fmt.Sprintf("Executing %s command from %s: %s", phase, phaseSource, command))
common.LogInfo1(fmt.Sprintf("Executing %s task from %s: %s", phase, phaseSource, command))
isHerokuishImage := common.IsImageHerokuishBased(image, appName)
script := constructScript(command, isHerokuishImage)
@@ -149,12 +151,22 @@ func executeScript(appName string, imageTag string, phase string) error {
}
var dockerArgs []string
if b, err := common.PlugnTriggerSetup("docker-args-deploy", []string{appName, imageTag}...).SetInput("").Output(); err != nil {
dockerArgs = append(dockerArgs, strings.Split(strings.TrimSpace(string(b[:])), "\n")...)
if b, err := common.PlugnTriggerSetup("docker-args-deploy", []string{appName, imageTag}...).SetInput("").Output(); err == nil {
words, err := shellquote.Split(strings.TrimSpace(string(b[:])))
if err != nil {
return err
}
dockerArgs = append(dockerArgs, words...)
}
if b, err := common.PlugnTriggerSetup("docker-args-process-deploy", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err != nil {
dockerArgs = append(dockerArgs, strings.Split(strings.TrimSpace(string(b[:])), "\n")...)
if b, err := common.PlugnTriggerSetup("docker-args-process-deploy", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err == nil {
words, err := shellquote.Split(strings.TrimSpace(string(b[:])))
if err != nil {
return err
}
dockerArgs = append(dockerArgs, words...)
}
filteredArgs := []string{"restart", "cpus", "memory", "memory-swap", "memory-reservation", "gpus"}
@@ -187,12 +199,22 @@ func executeScript(appName string, imageTag string, phase string) error {
}
containerCommand := []string{dokkuAppShell, "-c", script}
containerID, _ := createdContainerID(appName, dockerArgs, image, containerCommand, phase)
common.LogVerboseQuietContainerLogs(containerID)
if !waitForExecution(containerID) {
common.LogFail(fmt.Sprintf("Execution of '%s' command failed: %s", phase, command))
containerID, err := createdContainerID(appName, dockerArgs, image, containerCommand, phase)
if err != nil {
common.LogFail(fmt.Sprintf("Failed to create %s execution container: %s", phase, err.Error()))
}
if !waitForExecution(containerID) {
common.LogInfo2Quiet(fmt.Sprintf("Start of %s %s task (%s) output", appName, phase, containerID[0:9]))
common.LogVerboseQuietContainerLogs(containerID)
common.LogInfo2Quiet(fmt.Sprintf("End of %s %s task (%s) output", appName, phase, containerID[0:9]))
common.LogFail(fmt.Sprintf("Execution of %s task failed: %s", phase, command))
}
common.LogInfo2Quiet(fmt.Sprintf("Start of %s %s task (%s) output", appName, phase, containerID[0:9]))
common.LogVerboseQuietContainerLogs(containerID)
common.LogInfo2Quiet(fmt.Sprintf("End of %s %s task (%s) output", appName, phase, containerID[0:9]))
if phase != "predeploy" {
return nil
}
@@ -329,7 +351,7 @@ func createdContainerID(appName string, dockerArgs []string, image string, comma
b, err := containerCreateCmd.Output()
if err != nil {
return "", err
return "", errors.New(stderr.String())
}
containerID := strings.TrimSpace(string(b))

7
plugins/app-json/go.mod
View File

@@ -3,9 +3,10 @@ module github.com/dokku/dokku/plugins/app-json
go 1.14
require (
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect
)
replace github.com/dokku/dokku/plugins/common => ../common

2
plugins/app-json/go.sum
View File

@@ -2,5 +2,7 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg=
github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=

7
plugins/common/log.go
View File

@@ -81,7 +81,12 @@ func LogVerboseQuietContainerLogs(containerID string) {
LogExclaim(fmt.Sprintf("Failed to fetch container logs: %s", containerID))
}
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
output := strings.TrimSpace(string(b))
if len(output) == 0 {
return
}
for _, line := range strings.Split(output, "\n") {
if line != "" {
LogVerboseQuiet(line)
}

19
tests/apps/dockerfile-release/Dockerfile Normal file
View File

@@ -0,0 +1,19 @@
FROM python:3.8-buster
RUN pip install --user pipenv
ENV PATH="${PATH}:/root/.local/bin"
ENV PIPENV_VENV_IN_PROJECT=0
COPY . /app
WORKDIR /app
RUN ls -lah
RUN rm -rf .venv
RUN mkdir .venv
RUN pipenv install
CMD pipenv run gunicorn -b 0.0.0.0:8000 --workers=3 --threads=5 project.wsgi:application

17
tests/apps/dockerfile-release/Pipfile Normal file
View File

@@ -0,0 +1,17 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
django = "*"
gunicorn = "*"
djangorestframework = "*"
whitenoise = "*"
psycopg2-binary = "*"
dj-database-url = "*"
[requires]
python_version = "3.8"

120
tests/apps/dockerfile-release/Pipfile.lock generated Normal file
View File

@@ -0,0 +1,120 @@
{
"_meta": {
"hash": {
"sha256": "6e4ab3d777dd8ead8bc254ab4bb72ee80c93d1c09537b30ad23beb21c8bfdba9"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.8"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"asgiref": {
"hashes": [
"sha256:7e06d934a7718bf3975acbf87780ba678957b87c7adc056f13b6215d610695a0",
"sha256:ea448f92fc35a0ef4b1508f53a04c4670255a3f33d22a81c8fc9c872036adbe5"
],
"version": "==3.2.3"
},
"dj-database-url": {
"hashes": [
"sha256:4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163",
"sha256:851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9"
],
"index": "pypi",
"version": "==0.5.0"
},
"django": {
"hashes": [
"sha256:4f2c913303be4f874015993420bf0bd8fd2097a9c88e6b49c6a92f9bdd3fb13a",
"sha256:8c3575f81e11390893860d97e1e0154c47512f180ea55bd84ce8fa69ba8051ca"
],
"index": "pypi",
"version": "==3.0.2"
},
"djangorestframework": {
"hashes": [
"sha256:05809fc66e1c997fd9a32ea5730d9f4ba28b109b9da71fccfa5ff241201fd0a4",
"sha256:e782087823c47a26826ee5b6fa0c542968219263fb3976ec3c31edab23a4001f"
],
"index": "pypi",
"version": "==3.11.0"
},
"gunicorn": {
"hashes": [
"sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
"sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
],
"index": "pypi",
"version": "==20.0.4"
},
"psycopg2-binary": {
"hashes": [
"sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29",
"sha256:086f7e89ec85a6704db51f68f0dcae432eff9300809723a6e8782c41c2f48e03",
"sha256:18ca813fdb17bc1db73fe61b196b05dd1ca2165b884dd5ec5568877cabf9b039",
"sha256:19dc39616850342a2a6db70559af55b22955f86667b5f652f40c0e99253d9881",
"sha256:2166e770cb98f02ed5ee2b0b569d40db26788e0bf2ec3ae1a0d864ea6f1d8309",
"sha256:3a2522b1d9178575acee4adf8fd9f979f9c0449b00b4164bb63c3475ea6528ed",
"sha256:3aa773580f85a28ffdf6f862e59cb5a3cc7ef6885121f2de3fca8d6ada4dbf3b",
"sha256:3b5deaa3ee7180585a296af33e14c9b18c218d148e735c7accf78130765a47e3",
"sha256:407af6d7e46593415f216c7f56ba087a9a42bd6dc2ecb86028760aa45b802bd7",
"sha256:4c3c09fb674401f630626310bcaf6cd6285daf0d5e4c26d6e55ca26a2734e39b",
"sha256:4c6717962247445b4f9e21c962ea61d2e884fc17df5ddf5e35863b016f8a1f03",
"sha256:50446fae5681fc99f87e505d4e77c9407e683ab60c555ec302f9ac9bffa61103",
"sha256:5057669b6a66aa9ca118a2a860159f0ee3acf837eda937bdd2a64f3431361a2d",
"sha256:5dd90c5438b4f935c9d01fcbad3620253da89d19c1f5fca9158646407ed7df35",
"sha256:659c815b5b8e2a55193ede2795c1e2349b8011497310bb936da7d4745652823b",
"sha256:69b13fdf12878b10dc6003acc8d0abf3ad93e79813fd5f3812497c1c9fb9be49",
"sha256:7a1cb80e35e1ccea3e11a48afe65d38744a0e0bde88795cc56a4d05b6e4f9d70",
"sha256:7e6e3c52e6732c219c07bd97fff6c088f8df4dae3b79752ee3a817e6f32e177e",
"sha256:7f42a8490c4fe854325504ce7a6e4796b207960dabb2cbafe3c3959cb00d1d7e",
"sha256:84156313f258eafff716b2961644a4483a9be44a5d43551d554844d15d4d224e",
"sha256:8578d6b8192e4c805e85f187bc530d0f52ba86c39172e61cd51f68fddd648103",
"sha256:890167d5091279a27e2505ff0e1fb273f8c48c41d35c5b92adbf4af80e6b2ed6",
"sha256:98e10634792ac0e9e7a92a76b4991b44c2325d3e7798270a808407355e7bb0a1",
"sha256:9aadff9032e967865f9778485571e93908d27dab21d0fdfdec0ca779bb6f8ad9",
"sha256:9f24f383a298a0c0f9b3113b982e21751a8ecde6615494a3f1470eb4a9d70e9e",
"sha256:a73021b44813b5c84eda4a3af5826dd72356a900bac9bd9dd1f0f81ee1c22c2f",
"sha256:afd96845e12638d2c44d213d4810a08f4dc4a563f9a98204b7428e567014b1cd",
"sha256:b73ddf033d8cd4cc9dfed6324b1ad2a89ba52c410ef6877998422fcb9c23e3a8",
"sha256:b8f490f5fad1767a1331df1259763b3bad7d7af12a75b950c2843ba319b2415f",
"sha256:dbc5cd56fff1a6152ca59445178652756f4e509f672e49ccdf3d79c1043113a4",
"sha256:eac8a3499754790187bb00574ab980df13e754777d346f85e0ff6df929bcd964",
"sha256:eaed1c65f461a959284649e37b5051224f4db6ebdc84e40b5e65f2986f101a08"
],
"index": "pypi",
"version": "==2.8.4"
},
"pytz": {
"hashes": [
"sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
"sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
],
"version": "==2019.3"
},
"sqlparse": {
"hashes": [
"sha256:40afe6b8d4b1117e7dff5504d7a8ce07d9a1b15aeeade8a2d10f130a834f8177",
"sha256:7c3dca29c022744e95b547e867cee89f4fce4373f3549ccd8797d8eb52cdb873"
],
"version": "==0.3.0"
},
"whitenoise": {
"hashes": [
"sha256:0f9137f74bd95fa54329ace88d8dc695fbe895369a632e35f7a136e003e41d73",
"sha256:62556265ec1011bd87113fb81b7516f52688887b7a010ee899ff1fd18fd22700"
],
"index": "pypi",
"version": "==5.0.1"
}
},
"develop": {}
}

1
tests/apps/dockerfile-release/Procfile Normal file
View File

@@ -0,0 +1 @@
release: pipenv run python manage.py migrate

0
tests/apps/dockerfile-release/app/__init__.py Normal file
View File

3
tests/apps/dockerfile-release/app/admin.py Normal file
View File

@@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

5
tests/apps/dockerfile-release/app/apps.py Normal file
View File

@@ -0,0 +1,5 @@
from django.apps import AppConfig
class AppMainConfig(AppConfig):
name = 'app'

0
tests/apps/dockerfile-release/app/migrations/__init__.py Normal file
View File

3
tests/apps/dockerfile-release/app/models.py Normal file
View File

@@ -0,0 +1,3 @@
from django.db import models
# Create your models here.

3
tests/apps/dockerfile-release/app/tests.py Normal file
View File

@@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

3
tests/apps/dockerfile-release/app/views.py Normal file
View File

@@ -0,0 +1,3 @@
from django.shortcuts import render
# Create your views here.

11
tests/apps/dockerfile-release/manage.py Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
def main():
print("SECRET_KEY: {0}".format(os.getenv('SECRET_KEY')))
if __name__ == '__main__':
main()

0
tests/apps/dockerfile-release/project/__init__.py Normal file
View File

16
tests/apps/dockerfile-release/project/asgi.py Normal file
View File

@@ -0,0 +1,16 @@
"""
ASGI config for project project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')
application = get_asgi_application()

12
tests/apps/dockerfile-release/project/settings/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
import os
environment = os.getenv('ENVIRONMENT')
if environment == 'dev':
from .dev import *
elif environment == 'staging':
from .staging import *
elif environment == 'production':
from .staging import *
else:
raise Exception("Invalid environment")

135
tests/apps/dockerfile-release/project/settings/defaults.py Normal file
View File

@@ -0,0 +1,135 @@
"""
Django settings for project project.
Generated by 'django-admin startproject' using Django 3.0.2.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.0/ref/settings/
"""
import os
import dj_database_url
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.getenv('SECRET_KEY')
if not SECRET_KEY:
raise Exception("Secret Key Required")
ALLOWED_HOSTS = [
"*",
]
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'whitenoise.runserver_nostatic',
'django.contrib.staticfiles',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'whitenoise.middleware.WhiteNoiseMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'project.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'project.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
DATABASES = {
'default': dj_database_url.config(conn_max_age=500, ssl_require=True)
}
del DATABASES['default']['OPTIONS']['sslmode']
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
# Password validation
# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.0/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.0/howto/static-files/
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'staticfiles')
STATIC_URL = '/static/'
# Extra places for collectstatic to find static files.
STATICFILES_DIRS = [
os.path.join(PROJECT_ROOT, 'static'),
]
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

4
tests/apps/dockerfile-release/project/settings/dev.py Normal file
View File

@@ -0,0 +1,4 @@
from .defaults import *
DEBUG = True

1
tests/apps/dockerfile-release/project/settings/production.py Normal file
View File

@@ -0,0 +1 @@
from .defaults import *

1
tests/apps/dockerfile-release/project/settings/staging.py Normal file
View File

@@ -0,0 +1 @@
from .defaults import *

21
tests/apps/dockerfile-release/project/urls.py Normal file
View File

@@ -0,0 +1,21 @@
"""project URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
urlpatterns = [
path('admin/', admin.site.urls),
]

16
tests/apps/dockerfile-release/project/wsgi.py Normal file
View File

@@ -0,0 +1,16 @@
"""
WSGI config for project project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')
application = get_wsgi_application()

14
tests/unit/app-json.bats
View File

@@ -87,3 +87,17 @@ teardown() {
assert_success
assert_output '["./entrypoint"]'
}
@test "(app-json) app.json dockerfile release" {
run /bin/bash -c "dokku config:set --no-restart $TEST_APP SECRET_KEY=fjdkslafjdk ENVIRONMENT=dev"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app dockerfile-release
echo "output: $output"
echo "status: $status"
assert_output_contains "Executing release task from Procfile"
assert_output_contains "SECRET_KEY: fjdkslafjdk"
assert_success
}