Merge pull request #7572 from dokku/k3s-shm-size
Add support for setting shm-size for kubernetes deployments
This commit is contained in:
@@ -134,110 +134,42 @@ When attaching an worker or server node, the K3s plugin will look at the IP asso
|
|||||||
dokku scheduler-k3s:set --global network-interface eth1
|
dokku scheduler-k3s:set --global network-interface eth1
|
||||||
```
|
```
|
||||||
|
|
||||||
### Changing deploy timeouts
|
### Changing deployment settings
|
||||||
|
|
||||||
By default, app deploys will timeout after 300s. To customize this value, set the `deploy-timeout` property via `scheduler-k3s:set`:
|
The k3s plugin provides a number of settings that can be used to managed deployments on a per-app basis. The following table outlines ones not covered elsewhere:
|
||||||
|
|
||||||
|
| Name | Description | Global Default |
|
||||||
|
|-----------------------|---------------------------------------------------|--------------------|
|
||||||
|
| `deploy-timeout` | Controls when app deploys will timeout in seconds | `300s` |
|
||||||
|
| `image-pull-secrets` | Name of a kubernetes secret used to auth against a registry | Contents of `~/.docker/config.json` from Dokku server |
|
||||||
|
| `namespace` | Controls the namespace used for resource creation | `default` |
|
||||||
|
| `rollback-on-failure` | Whether to rollback failed deploys | `false` |
|
||||||
|
| `shm-size` | Default shared memory size for pods | Kubernetes default |
|
||||||
|
|
||||||
|
All settings can be set via the `scheduler-k3s:set` command. Using `deploy-timeout` as an example:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
dokku scheduler-k3s:set node-js-app deploy-timeout 60s
|
dokku scheduler-k3s:set node-js-app deploy-timeout 60s
|
||||||
```
|
```
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option:
|
The default value may be set by passing an empty value for the option in question:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
dokku scheduler-k3s:set node-js-app deploy-timeout
|
dokku scheduler-k3s:set node-js-app deploy-timeout
|
||||||
```
|
```
|
||||||
|
|
||||||
The `deploy-timeout` property can also be set globally. The global default is `300s`.
|
Properties can also be set globally. If not set for an app, the global value will apply.
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
dokku scheduler-k3s:set --global deploy-timeout 60s
|
dokku scheduler-k3s:set --global deploy-timeout 60s
|
||||||
```
|
```
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option.
|
The global default value may be set by passing an empty value for the option.
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
dokku scheduler-k3s:set --global deploy-timeout
|
dokku scheduler-k3s:set --global deploy-timeout
|
||||||
```
|
```
|
||||||
|
|
||||||
### Customizing the namespace
|
|
||||||
|
|
||||||
By default, app deploys will run against the `default` Kubernetes namespace. To customize this value, set the `namespace` property via `scheduler-k3s:set`:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set node-js-app namespace lollipop
|
|
||||||
```
|
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set node-js-app namespace
|
|
||||||
```
|
|
||||||
|
|
||||||
The `namespace` property can also be set globally. The global default is `default`.
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set --global namespace 60s
|
|
||||||
```
|
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option.
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set --global namespace
|
|
||||||
```
|
|
||||||
|
|
||||||
### Enabling rollback on failure
|
|
||||||
|
|
||||||
By default, app deploys do not rollback on failure. To enable this functionality, set the `rollback-on-failure` property via `scheduler-k3s:set`:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set node-js-app rollback-on-failure true
|
|
||||||
```
|
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set node-js-app rollback-on-failure
|
|
||||||
```
|
|
||||||
|
|
||||||
The `rollback-on-failure` property can also be set globally. The global default is `false`.
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set --global rollback-on-failure false
|
|
||||||
```
|
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option.
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set --global rollback-on-failure
|
|
||||||
```
|
|
||||||
|
|
||||||
### Using image pull secrets
|
|
||||||
|
|
||||||
When authenticating against a registry via `registry:login`, the scheduler-k3s plugin will authenticate all servers in the cluster against the registry specified. If desired, an image pull secret can be used instead. To customize this value, set the `image-pull-secrets` property via `scheduler-k3s:set`:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set node-js-app image-pull-secrets lollipop
|
|
||||||
```
|
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set node-js-app image-pull-secrets
|
|
||||||
```
|
|
||||||
|
|
||||||
The `image-pull-secrets` property can also be set globally. The global default is empty string, and k3s will use Dokku's locally configured `~/.docker/config.json` for any private registry pulls.
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set --global image-pull-secrets 60s
|
|
||||||
```
|
|
||||||
|
|
||||||
The default value may be set by passing an empty value for the option.
|
|
||||||
|
|
||||||
```shell
|
|
||||||
dokku scheduler-k3s:set --global image-pull-secrets
|
|
||||||
```
|
|
||||||
|
|
||||||
### SSL Certificates
|
### SSL Certificates
|
||||||
|
|
||||||
#### Enabling letsencrypt integration
|
#### Enabling letsencrypt integration
|
||||||
|
|||||||
@@ -1227,6 +1227,23 @@ func getComputedRollbackOnFailure(appName string) string {
|
|||||||
return rollbackOnFailure
|
return rollbackOnFailure
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getShmSize(appName string) string {
|
||||||
|
return common.PropertyGetDefault("scheduler-k3s", appName, "shm-size", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func getGlobalShmSize() string {
|
||||||
|
return common.PropertyGetDefault("scheduler-k3s", "--global", "shm-size", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func getComputedShmSize(appName string) string {
|
||||||
|
shmSize := getShmSize(appName)
|
||||||
|
if shmSize == "" {
|
||||||
|
shmSize = getGlobalShmSize()
|
||||||
|
}
|
||||||
|
|
||||||
|
return shmSize
|
||||||
|
}
|
||||||
|
|
||||||
func getGlobalGlobalToken() string {
|
func getGlobalGlobalToken() string {
|
||||||
return common.PropertyGet("scheduler-k3s", "--global", "token")
|
return common.PropertyGet("scheduler-k3s", "--global", "token")
|
||||||
}
|
}
|
||||||
@@ -1436,7 +1453,7 @@ func getServerIP() (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(serverIP) == 0 {
|
if len(serverIP) == 0 {
|
||||||
return "", fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface))
|
return "", fmt.Errorf("Unable to determine server ip address from network-interface %s", networkInterface)
|
||||||
}
|
}
|
||||||
return serverIP, nil
|
return serverIP, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,6 +35,9 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
|
|||||||
"--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure,
|
"--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure,
|
||||||
"--scheduler-k3s-rollback-on-failure": reportRollbackOnFailure,
|
"--scheduler-k3s-rollback-on-failure": reportRollbackOnFailure,
|
||||||
"--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure,
|
"--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure,
|
||||||
|
"--scheduler-k3s-computed-shm-size": reportComputedShmSize,
|
||||||
|
"--scheduler-k3s-global-shm-size": reportGlobalShmSize,
|
||||||
|
"--scheduler-k3s-shm-size": reportShmSize,
|
||||||
}
|
}
|
||||||
|
|
||||||
chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart.")
|
chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart.")
|
||||||
@@ -172,3 +175,15 @@ func reportRollbackOnFailure(appName string) string {
|
|||||||
func reportGlobalRollbackOnFailure(appName string) string {
|
func reportGlobalRollbackOnFailure(appName string) string {
|
||||||
return getGlobalRollbackOnFailure()
|
return getGlobalRollbackOnFailure()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func reportComputedShmSize(appName string) string {
|
||||||
|
return getComputedShmSize(appName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportGlobalShmSize(appName string) string {
|
||||||
|
return getGlobalShmSize()
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportShmSize(appName string) string {
|
||||||
|
return getShmSize(appName)
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ var (
|
|||||||
"image-pull-secrets": "",
|
"image-pull-secrets": "",
|
||||||
"namespace": "",
|
"namespace": "",
|
||||||
"rollback-on-failure": "",
|
"rollback-on-failure": "",
|
||||||
|
"shm-size": "",
|
||||||
}
|
}
|
||||||
|
|
||||||
// GlobalProperties is a map of all valid global k3s properties
|
// GlobalProperties is a map of all valid global k3s properties
|
||||||
@@ -38,6 +39,7 @@ var (
|
|||||||
"namespace": true,
|
"namespace": true,
|
||||||
"network-interface": true,
|
"network-interface": true,
|
||||||
"rollback-on-failure": true,
|
"rollback-on-failure": true,
|
||||||
|
"shm-size": true,
|
||||||
"token": true,
|
"token": true,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -113,6 +113,18 @@ type ProcessValues struct {
|
|||||||
Replicas int32 `yaml:"replicas"`
|
Replicas int32 `yaml:"replicas"`
|
||||||
Resources ProcessResourcesMap `yaml:"resources,omitempty"`
|
Resources ProcessResourcesMap `yaml:"resources,omitempty"`
|
||||||
Web ProcessWeb `yaml:"web,omitempty"`
|
Web ProcessWeb `yaml:"web,omitempty"`
|
||||||
|
Volumes []ProcessVolume `yaml:"volumes,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProcessVolume struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
MountPath string `yaml:"mount_path"`
|
||||||
|
EmptyDir *ProcessVolumeEmptyDir `yaml:"empty_dir,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProcessVolumeEmptyDir struct {
|
||||||
|
Medium string `yaml:"medium"`
|
||||||
|
SizeLimit string `yaml:"size_limit"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProcessAnnotations struct {
|
type ProcessAnnotations struct {
|
||||||
|
|||||||
@@ -116,8 +116,30 @@ spec:
|
|||||||
{{- if $.Values.global.image.working_dir }}
|
{{- if $.Values.global.image.working_dir }}
|
||||||
workingDir: {{ $.Values.global.image.working_dir }}
|
workingDir: {{ $.Values.global.image.working_dir }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- if $config.volumes }}
|
||||||
|
volumeMounts:
|
||||||
|
{{- range $volume := $config.volumes }}
|
||||||
|
- name: {{ $volume.name }}
|
||||||
|
mountPath: {{ $volume.mount_path }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
{{- if $.Values.global.image.image_pull_secrets }}
|
{{- if $.Values.global.image.image_pull_secrets }}
|
||||||
imagePullSecrets:
|
imagePullSecrets:
|
||||||
- name: {{ $.Values.global.image.image_pull_secrets }}
|
- name: {{ $.Values.global.image.image_pull_secrets }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
serviceAccountName: {{ $.Values.global.app_name }}
|
serviceAccountName: {{ $.Values.global.app_name }}
|
||||||
|
{{- if $config.volumes }}
|
||||||
|
volumes:
|
||||||
|
{{- range $volume := $config.volumes }}
|
||||||
|
- name: {{ $volume.name }}
|
||||||
|
{{- if $volume.empty_dir }}
|
||||||
|
emptyDir:
|
||||||
|
{{- if $volume.empty_dir.medium }}
|
||||||
|
medium: {{ $volume.empty_dir.medium }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if $volume.empty_dir.size_limit }}
|
||||||
|
sizeLimit: {{ $volume.empty_dir.size_limit }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|||||||
@@ -382,6 +382,18 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
processVolumes := []ProcessVolume{}
|
||||||
|
if shmSize := getComputedShmSize(appName); shmSize != "" {
|
||||||
|
processVolumes = append(processVolumes, ProcessVolume{
|
||||||
|
Name: "shmem",
|
||||||
|
MountPath: "/dev/shm",
|
||||||
|
EmptyDir: &ProcessVolumeEmptyDir{
|
||||||
|
Medium: "Memory",
|
||||||
|
SizeLimit: shmSize,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
for processType, processCount := range processes {
|
for processType, processCount := range processes {
|
||||||
// todo: implement deployment annotations
|
// todo: implement deployment annotations
|
||||||
// todo: implement pod annotations
|
// todo: implement pod annotations
|
||||||
@@ -439,6 +451,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
|
|||||||
ProcessType: ProcessType_Worker,
|
ProcessType: ProcessType_Worker,
|
||||||
Replicas: int32(processCount),
|
Replicas: int32(processCount),
|
||||||
Resources: processResources,
|
Resources: processResources,
|
||||||
|
Volumes: processVolumes,
|
||||||
}
|
}
|
||||||
|
|
||||||
if processType == "web" {
|
if processType == "web" {
|
||||||
@@ -590,6 +603,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
|
|||||||
ProcessType: ProcessType_Cron,
|
ProcessType: ProcessType_Cron,
|
||||||
Replicas: 1,
|
Replicas: 1,
|
||||||
Resources: processResources,
|
Resources: processResources,
|
||||||
|
Volumes: processVolumes,
|
||||||
}
|
}
|
||||||
values.Processes[cronEntry.ID] = processValues
|
values.Processes[cronEntry.ID] = processValues
|
||||||
|
|
||||||
|
|||||||
@@ -179,12 +179,29 @@ teardown_() {
|
|||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku scheduler-k3s:set $TEST_APP shm-size 64Mi"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git"
|
run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git"
|
||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
|
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.volumes[0].emptyDir.sizeLimit'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "64Mi"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.volumes[0].emptyDir.medium'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "Memory"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "(scheduler-k3s) deploy annotations" {
|
@test "(scheduler-k3s) deploy annotations" {
|
||||||
|
|||||||
Reference in New Issue
Block a user