github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #7975 from dokku/7664-k3s-security-context

Browse Source 
Create SecurityContext for k3s scheduler from docker-options
This commit is contained in:
Jose Diaz-Gonzalez
2025-09-14 03:10:52 -04:00
committed by GitHub
parent fa0c9663cf ef798780c6
commit 408417fe2e
8 changed files with 264 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/deployment/schedulers/k3s.md
View File

@@ -568,6 +568,11 @@ This plugin implements various functionality through `plugn` triggers to integra
- `apps:clone`
- `apps:destroy`
- `apps:rename`
- `docker-options`:
- The following docker options are translated into their kubernetes equivalents:
- `--cap-add`
- `--cap-drop`
- `--privileged`
- `cron`
- `enter`
- `deploy`

56
plugins/docker-options/dockeroptions.go
View File

@@ -6,6 +6,8 @@ import (
"os"
"sort"
"strings"
"github.com/dokku/dokku/plugins/common"
)
// SetDockerOptionForPhases sets an option to specified phases
@@ -85,3 +87,57 @@ func GetDockerOptionsForPhase(appName string, phase string) ([]string, error) {
return options, nil
}
// GetSpecifiedDockerOptionsForPhase returns the docker options for the specified phase that are in the desiredOptions list
// It expects desiredOptions to be a list of docker options that are in the format "--option"
// And will retrieve any lines that start with the desired option
func GetSpecifiedDockerOptionsForPhase(appName string, phase string, desiredOptions []string) (map[string][]string, error) {
foundOptions := map[string][]string{}
options, err := GetDockerOptionsForPhase(appName, phase)
if err != nil {
return foundOptions, err
}
for _, option := range options {
for _, desiredOption := range desiredOptions {
if option == desiredOption {
foundOptions[desiredOption] = []string{}
break
}
// match options that are in the format "--option=value"
if strings.HasPrefix(option, fmt.Sprintf("%s=", desiredOption)) {
if _, ok := foundOptions[desiredOption]; !ok {
foundOptions[desiredOption] = []string{}
}
parts := strings.SplitN(option, "=", 2)
if len(parts) != 2 {
common.LogWarn(fmt.Sprintf("Invalid docker option found for %s: %s", appName, option))
continue
}
foundOptions[desiredOption] = append(foundOptions[desiredOption], parts[1])
break
}
// match options that are in the format "--option value"
if strings.HasPrefix(option, fmt.Sprintf("%s ", desiredOption)) {
if _, ok := foundOptions[desiredOption]; !ok {
foundOptions[desiredOption] = []string{}
}
parts := strings.SplitN(option, " ", 2)
if len(parts) != 2 {
common.LogWarn(fmt.Sprintf("Invalid docker option found for %s: %s", appName, option))
continue
}
foundOptions[desiredOption] = append(foundOptions[desiredOption], parts[1])
break
}
}
}
return foundOptions, nil
}

32
plugins/scheduler-k3s/functions.go
View File

@@ -18,6 +18,7 @@ import (
appjson "github.com/dokku/dokku/plugins/app-json"
"github.com/dokku/dokku/plugins/common"
dockeroptions "github.com/dokku/dokku/plugins/docker-options"
"github.com/dokku/dokku/plugins/logs"
nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts"
resty "github.com/go-resty/resty/v2"
@@ -1489,6 +1490,37 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
}, nil
}
func getSecurityContext(appName string, phase string) (SecurityContext, error) {
securityContext := SecurityContext{}
deployOptions, err := dockeroptions.GetSpecifiedDockerOptionsForPhase(appName, phase, []string{
"--cap-add",
"--cap-drop",
"--privileged",
})
if err != nil {
return SecurityContext{}, fmt.Errorf("Error getting deploy options: %w", err)
}
if _, ok := deployOptions["--privileged"]; ok {
securityContext.Privileged = true
}
if capAdd, ok := deployOptions["--cap-add"]; ok {
capabilities := []string{}
for _, cap := range capAdd {
capabilities = append(capabilities, strings.ToUpper(cap))
}
securityContext.Capabilities.Add = capabilities
}
if capDrop, ok := deployOptions["--cap-drop"]; ok {
capabilities := []string{}
for _, cap := range capDrop {
capabilities = append(capabilities, strings.ToUpper(cap))
}
securityContext.Capabilities.Drop = capabilities
}
return securityContext, nil
}
func getProcessSpecificKustomizeRootPath(appName string) string {
if !hasKustomizeDirectory(appName) {
return ""

71
plugins/scheduler-k3s/template.go
View File

@@ -3,6 +3,7 @@ package scheduler_k3s
import (
"crypto/rand"
"fmt"
"maps"
"os"
"strings"
@@ -41,15 +42,16 @@ type AppValues struct {
}
type GlobalValues struct {
Annotations ProcessAnnotations `yaml:"annotations,omitempty"`
AppName string `yaml:"app_name"`
DeploymentID string `yaml:"deployment_id"`
Image GlobalImage `yaml:"image"`
Labels ProcessLabels `yaml:"labels,omitempty"`
Keda GlobalKedaValues `yaml:"keda"`
Namespace string `yaml:"namespace"`
Network GlobalNetwork `yaml:"network"`
Secrets map[string]string `yaml:"secrets,omitempty"`
Annotations ProcessAnnotations `yaml:"annotations,omitempty"`
AppName string `yaml:"app_name"`
DeploymentID string `yaml:"deployment_id"`
Image GlobalImage `yaml:"image"`
Labels ProcessLabels `yaml:"labels,omitempty"`
Keda GlobalKedaValues `yaml:"keda"`
Namespace string `yaml:"namespace"`
Network GlobalNetwork `yaml:"network"`
Secrets map[string]string `yaml:"secrets,omitempty"`
SecurityContext SecurityContext `yaml:"security_context,omitempty"`
}
type GlobalImage struct {
@@ -351,11 +353,53 @@ type Job struct {
Namespace string
ProcessType string
Schedule string
SecurityContext SecurityContext
Suffix string
RemoveContainer bool
WorkingDir string
}
// SecurityContext contains the security context for a process
type SecurityContext struct {
// Capabilities contains the capabilities for a process
Capabilities SecurityContextCapabilities `yaml:"capabilities,omitempty"`
// Privileged contains the privileged flag for a process
Privileged bool `yaml:"privileged,omitempty"`
}
// ToCoreV1SecurityContext converts the security context to a corev1.SecurityContext
func (s SecurityContext) ToCoreV1SecurityContext() corev1.SecurityContext {
securityContext := corev1.SecurityContext{
Capabilities: &corev1.Capabilities{},
Privileged: ptr.To(s.Privileged),
}
if len(s.Capabilities.Add) > 0 {
capabilities := make([]corev1.Capability, len(s.Capabilities.Add))
for i, cap := range s.Capabilities.Add {
capabilities[i] = corev1.Capability(cap)
}
securityContext.Capabilities.Add = capabilities
}
if len(s.Capabilities.Drop) > 0 {
capabilities := make([]corev1.Capability, len(s.Capabilities.Drop))
for i, cap := range s.Capabilities.Drop {
capabilities[i] = corev1.Capability(cap)
}
securityContext.Capabilities.Drop = capabilities
}
return securityContext
}
// SecurityContextCapabilities contains the capabilities for a process
type SecurityContextCapabilities struct {
// Add contains the add capabilities for a process
Add []string `yaml:"add,omitempty"`
// Drop contains the drop capabilities for a process
Drop []string `yaml:"drop,omitempty"`
}
func templateKubernetesJob(input Job) (batchv1.Job, error) {
labels := map[string]string{
"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
@@ -368,9 +412,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
"dokku.com/managed": "true",
}
for key, value := range input.Labels {
labels[key] = value
}
maps.Copy(labels, input.Labels)
secretName := fmt.Sprintf("env-%s.%d", input.AppName, input.DeploymentID)
env := []corev1.EnvVar{}
@@ -417,6 +459,8 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
podAnnotations[key] = value
}
securityContext := input.SecurityContext.ToCoreV1SecurityContext()
job := batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
@@ -453,7 +497,8 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
},
WorkingDir: input.WorkingDir,
SecurityContext: &securityContext,
WorkingDir: input.WorkingDir,
},
},
RestartPolicy: corev1.RestartPolicyNever,

21
plugins/scheduler-k3s/templates/chart/cron-job.yaml
View File

@@ -102,6 +102,27 @@ spec:
{{- end }}
{{- end }}
{{- end }}
{{- if hasKey $.Values.global "security_context" }}
securityContext:
{{- if $.Values.global.security_context.privileged }}
privileged: true
{{- end }}
{{- if hasKey $.Values.global.security_context "capabilities" }}
capabilities:
{{- if hasKey $.Values.global.security_context.capabilities "add" }}
add:
{{- range $.Values.global.security_context.capabilities.add }}
- {{ . }}
{{- end }}
{{- end }}
{{- if hasKey $.Values.global.security_context.capabilities "drop" }}
drop:
{{- range $.Values.global.security_context.capabilities.drop }}
- {{ . }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if $.Values.global.image.working_dir }}
workingDir: {{ $.Values.global.image.working_dir }}
{{- end }}

21
plugins/scheduler-k3s/templates/chart/deployment.yaml
View File

@@ -117,6 +117,27 @@ spec:
readinessProbe:
{{ $config.healthchecks.readiness | toJson | indent 10 }}
{{- end }}
{{- if hasKey $.Values.global "security_context" }}
securityContext:
{{- if $.Values.global.security_context.privileged }}
privileged: true
{{- end }}
{{- if hasKey $.Values.global.security_context "capabilities" }}
capabilities:
{{- if hasKey $.Values.global.security_context.capabilities "add" }}
add:
{{- range $.Values.global.security_context.capabilities.add }}
- {{ . }}
{{- end }}
{{- end }}
{{- if hasKey $.Values.global.security_context.capabilities "drop" }}
drop:
{{- range $.Values.global.security_context.capabilities.drop }}
- {{ . }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if $.Values.global.image.working_dir }}
workingDir: {{ $.Values.global.image.working_dir }}
{{- end }}

14
plugins/scheduler-k3s/triggers.go
View File

@@ -387,6 +387,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
return fmt.Errorf("Error getting keda values: %w", err)
}
securityContext, err := getSecurityContext(appName, "deploy")
if err != nil {
return fmt.Errorf("Error getting security context: %w", err)
}
values := &AppValues{
Global: GlobalValues{
Annotations: globalAnnotations,
@@ -407,7 +412,8 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
PrimaryPort: primaryPort,
PrimaryServicePort: primaryServicePort,
},
Secrets: map[string]string{},
Secrets: map[string]string{},
SecurityContext: securityContext,
},
Processes: map[string]ProcessValues{},
}
@@ -1228,6 +1234,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
}
}
securityContext, err := getSecurityContext(appName, "run")
if err != nil {
return fmt.Errorf("Error getting security context: %w", err)
}
workingDir := common.GetWorkingDir(appName, image)
job, err := templateKubernetesJob(Job{
AppName: appName,
@@ -1243,6 +1254,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
Namespace: namespace,
ProcessType: processType,
RemoveContainer: rmContainer,
SecurityContext: securityContext,
WorkingDir: workingDir,
})
if err != nil {

59
tests/unit/scheduler-k3s-3.bats
View File

@@ -11,12 +11,69 @@ setup() {
export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
}
teardown_() {
teardown() {
global_teardown
dokku nginx:start
uninstall_k3s || true
}
@test "(scheduler-k3s) security context" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi
INGRESS_CLASS=nginx install_k3s
run /bin/bash -c "dokku apps:create $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "null"
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.privileged'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "null"
run /bin/bash -c "dokku docker-options:add $TEST_APP deploy --cap-add=NET_ADMIN"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku docker-options:add $TEST_APP deploy --privileged"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:restart $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "NET_ADMIN"
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.privileged'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "true"
}
@test "(scheduler-k3s) kustomize" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"