diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 34a6106b7..f66026081 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -134,110 +134,42 @@ When attaching an worker or server node, the K3s plugin will look at the IP asso dokku scheduler-k3s:set --global network-interface eth1 ``` -### Changing deploy timeouts +### Changing deployment settings -By default, app deploys will timeout after 300s. To customize this value, set the `deploy-timeout` property via `scheduler-k3s:set`: +The k3s plugin provides a number of settings that can be used to managed deployments on a per-app basis. The following table outlines ones not covered elsewhere: + +| Name | Description | Global Default | +|-----------------------|---------------------------------------------------|--------------------| +| `deploy-timeout` | Controls when app deploys will timeout in seconds | `300s` | +| `image-pull-secrets` | Name of a kubernetes secret used to auth against a registry | Contents of `~/.docker/config.json` from Dokku server | +| `namespace` | Controls the namespace used for resource creation | `default` | +| `rollback-on-failure` | Whether to rollback failed deploys | `false` | +| `shm-size` | Default shared memory size for pods | Kubernetes default | + +All settings can be set via the `scheduler-k3s:set` command. Using `deploy-timeout` as an example: ```shell dokku scheduler-k3s:set node-js-app deploy-timeout 60s ``` -The default value may be set by passing an empty value for the option: +The default value may be set by passing an empty value for the option in question: ```shell dokku scheduler-k3s:set node-js-app deploy-timeout ``` -The `deploy-timeout` property can also be set globally. The global default is `300s`. +Properties can also be set globally. If not set for an app, the global value will apply. ```shell dokku scheduler-k3s:set --global deploy-timeout 60s ``` -The default value may be set by passing an empty value for the option. +The global default value may be set by passing an empty value for the option. ```shell dokku scheduler-k3s:set --global deploy-timeout ``` -### Customizing the namespace - -By default, app deploys will run against the `default` Kubernetes namespace. To customize this value, set the `namespace` property via `scheduler-k3s:set`: - -```shell -dokku scheduler-k3s:set node-js-app namespace lollipop -``` - -The default value may be set by passing an empty value for the option: - -```shell -dokku scheduler-k3s:set node-js-app namespace -``` - -The `namespace` property can also be set globally. The global default is `default`. - -```shell -dokku scheduler-k3s:set --global namespace 60s -``` - -The default value may be set by passing an empty value for the option. - -```shell -dokku scheduler-k3s:set --global namespace -``` - -### Enabling rollback on failure - -By default, app deploys do not rollback on failure. To enable this functionality, set the `rollback-on-failure` property via `scheduler-k3s:set`: - -```shell -dokku scheduler-k3s:set node-js-app rollback-on-failure true -``` - -The default value may be set by passing an empty value for the option: - -```shell -dokku scheduler-k3s:set node-js-app rollback-on-failure -``` - -The `rollback-on-failure` property can also be set globally. The global default is `false`. - -```shell -dokku scheduler-k3s:set --global rollback-on-failure false -``` - -The default value may be set by passing an empty value for the option. - -```shell -dokku scheduler-k3s:set --global rollback-on-failure -``` - -### Using image pull secrets - -When authenticating against a registry via `registry:login`, the scheduler-k3s plugin will authenticate all servers in the cluster against the registry specified. If desired, an image pull secret can be used instead. To customize this value, set the `image-pull-secrets` property via `scheduler-k3s:set`: - -```shell -dokku scheduler-k3s:set node-js-app image-pull-secrets lollipop -``` - -The default value may be set by passing an empty value for the option: - -```shell -dokku scheduler-k3s:set node-js-app image-pull-secrets -``` - -The `image-pull-secrets` property can also be set globally. The global default is empty string, and k3s will use Dokku's locally configured `~/.docker/config.json` for any private registry pulls. - -```shell -dokku scheduler-k3s:set --global image-pull-secrets 60s -``` - -The default value may be set by passing an empty value for the option. - -```shell -dokku scheduler-k3s:set --global image-pull-secrets -``` - ### SSL Certificates #### Enabling letsencrypt integration diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index f03b9159c..50e5105e2 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1227,6 +1227,23 @@ func getComputedRollbackOnFailure(appName string) string { return rollbackOnFailure } +func getShmSize(appName string) string { + return common.PropertyGetDefault("scheduler-k3s", appName, "shm-size", "") +} + +func getGlobalShmSize() string { + return common.PropertyGetDefault("scheduler-k3s", "--global", "shm-size", "") +} + +func getComputedShmSize(appName string) string { + shmSize := getShmSize(appName) + if shmSize == "" { + shmSize = getGlobalShmSize() + } + + return shmSize +} + func getGlobalGlobalToken() string { return common.PropertyGet("scheduler-k3s", "--global", "token") } @@ -1436,7 +1453,7 @@ func getServerIP() (string, error) { } if len(serverIP) == 0 { - return "", fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface)) + return "", fmt.Errorf("Unable to determine server ip address from network-interface %s", networkInterface) } return serverIP, nil } diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index 4e136eea5..d480969b6 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -35,6 +35,9 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure, "--scheduler-k3s-rollback-on-failure": reportRollbackOnFailure, "--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure, + "--scheduler-k3s-computed-shm-size": reportComputedShmSize, + "--scheduler-k3s-global-shm-size": reportGlobalShmSize, + "--scheduler-k3s-shm-size": reportShmSize, } chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart.") @@ -172,3 +175,15 @@ func reportRollbackOnFailure(appName string) string { func reportGlobalRollbackOnFailure(appName string) string { return getGlobalRollbackOnFailure() } + +func reportComputedShmSize(appName string) string { + return getComputedShmSize(appName) +} + +func reportGlobalShmSize(appName string) string { + return getGlobalShmSize() +} + +func reportShmSize(appName string) string { + return getShmSize(appName) +} diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index c4e6aea34..811618cbd 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -23,6 +23,7 @@ var ( "image-pull-secrets": "", "namespace": "", "rollback-on-failure": "", + "shm-size": "", } // GlobalProperties is a map of all valid global k3s properties @@ -38,6 +39,7 @@ var ( "namespace": true, "network-interface": true, "rollback-on-failure": true, + "shm-size": true, "token": true, } ) diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 94bb9baa8..8ce060806 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -113,6 +113,18 @@ type ProcessValues struct { Replicas int32 `yaml:"replicas"` Resources ProcessResourcesMap `yaml:"resources,omitempty"` Web ProcessWeb `yaml:"web,omitempty"` + Volumes []ProcessVolume `yaml:"volumes,omitempty"` +} + +type ProcessVolume struct { + Name string `yaml:"name"` + MountPath string `yaml:"mount_path"` + EmptyDir *ProcessVolumeEmptyDir `yaml:"empty_dir,omitempty"` +} + +type ProcessVolumeEmptyDir struct { + Medium string `yaml:"medium"` + SizeLimit string `yaml:"size_limit"` } type ProcessAnnotations struct { diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index 1457ccd5f..3bb1d6e64 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -116,8 +116,30 @@ spec: {{- if $.Values.global.image.working_dir }} workingDir: {{ $.Values.global.image.working_dir }} {{- end }} + {{- if $config.volumes }} + volumeMounts: + {{- range $volume := $config.volumes }} + - name: {{ $volume.name }} + mountPath: {{ $volume.mount_path }} + {{- end }} + {{- end }} {{- if $.Values.global.image.image_pull_secrets }} imagePullSecrets: - name: {{ $.Values.global.image.image_pull_secrets }} {{- end }} serviceAccountName: {{ $.Values.global.app_name }} + {{- if $config.volumes }} + volumes: + {{- range $volume := $config.volumes }} + - name: {{ $volume.name }} + {{- if $volume.empty_dir }} + emptyDir: + {{- if $volume.empty_dir.medium }} + medium: {{ $volume.empty_dir.medium }} + {{- end }} + {{- if $volume.empty_dir.size_limit }} + sizeLimit: {{ $volume.empty_dir.size_limit }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index aede9c473..cb4c27e49 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -382,6 +382,18 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } } + processVolumes := []ProcessVolume{} + if shmSize := getComputedShmSize(appName); shmSize != "" { + processVolumes = append(processVolumes, ProcessVolume{ + Name: "shmem", + MountPath: "/dev/shm", + EmptyDir: &ProcessVolumeEmptyDir{ + Medium: "Memory", + SizeLimit: shmSize, + }, + }) + } + for processType, processCount := range processes { // todo: implement deployment annotations // todo: implement pod annotations @@ -439,6 +451,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e ProcessType: ProcessType_Worker, Replicas: int32(processCount), Resources: processResources, + Volumes: processVolumes, } if processType == "web" { @@ -590,6 +603,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e ProcessType: ProcessType_Cron, Replicas: 1, Resources: processResources, + Volumes: processVolumes, } values.Processes[cronEntry.ID] = processValues diff --git a/tests/unit/scheduler-k3s-2.bats b/tests/unit/scheduler-k3s-2.bats index 676f5612f..60734ad07 100644 --- a/tests/unit/scheduler-k3s-2.bats +++ b/tests/unit/scheduler-k3s-2.bats @@ -179,12 +179,29 @@ teardown_() { echo "status: $status" assert_success + run /bin/bash -c "dokku scheduler-k3s:set $TEST_APP shm-size 64Mi" + echo "output: $output" + echo "status: $status" + assert_success + run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git" echo "output: $output" echo "status: $status" assert_success assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.volumes[0].emptyDir.sizeLimit'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "64Mi" + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.volumes[0].emptyDir.medium'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "Memory" } @test "(scheduler-k3s) deploy annotations" {