Merge pull request #7975 from dokku/7664-k3s-security-context

Create SecurityContext for k3s scheduler from docker-options

docs/deployment/schedulers/k3s.md

@@ -568,6 +568,11 @@ This plugin implements various functionality through `plugn` triggers to integra
 - `apps:clone`
 - `apps:destroy`
 - `apps:rename`
+- `docker-options`:
+    - The following docker options are translated into their kubernetes equivalents:
+        - `--cap-add`
+        - `--cap-drop`
+        - `--privileged`
 - `cron`
 - `enter`
 - `deploy`

plugins/docker-options/dockeroptions.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"sort"
 	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
 )
 
 // SetDockerOptionForPhases sets an option to specified phases
@@ -85,3 +87,57 @@ func GetDockerOptionsForPhase(appName string, phase string) ([]string, error) {
 
 	return options, nil
 }
+
+// GetSpecifiedDockerOptionsForPhase returns the docker options for the specified phase that are in the desiredOptions list
+// It expects desiredOptions to be a list of docker options that are in the format "--option"
+// And will retrieve any lines that start with the desired option
+func GetSpecifiedDockerOptionsForPhase(appName string, phase string, desiredOptions []string) (map[string][]string, error) {
+	foundOptions := map[string][]string{}
+	options, err := GetDockerOptionsForPhase(appName, phase)
+	if err != nil {
+		return foundOptions, err
+	}
+
+	for _, option := range options {
+		for _, desiredOption := range desiredOptions {
+			if option == desiredOption {
+				foundOptions[desiredOption] = []string{}
+				break
+			}
+
+			// match options that are in the format "--option=value"
+			if strings.HasPrefix(option, fmt.Sprintf("%s=", desiredOption)) {
+				if _, ok := foundOptions[desiredOption]; !ok {
+					foundOptions[desiredOption] = []string{}
+				}
+
+				parts := strings.SplitN(option, "=", 2)
+				if len(parts) != 2 {
+					common.LogWarn(fmt.Sprintf("Invalid docker option found for %s: %s", appName, option))
+					continue
+				}
+
+				foundOptions[desiredOption] = append(foundOptions[desiredOption], parts[1])
+				break
+			}
+
+			// match options that are in the format "--option value"
+			if strings.HasPrefix(option, fmt.Sprintf("%s ", desiredOption)) {
+				if _, ok := foundOptions[desiredOption]; !ok {
+					foundOptions[desiredOption] = []string{}
+				}
+
+				parts := strings.SplitN(option, " ", 2)
+				if len(parts) != 2 {
+					common.LogWarn(fmt.Sprintf("Invalid docker option found for %s: %s", appName, option))
+					continue
+				}
+
+				foundOptions[desiredOption] = append(foundOptions[desiredOption], parts[1])
+				break
+			}
+		}
+	}
+
+	return foundOptions, nil
+}

plugins/scheduler-k3s/functions.go

@@ -18,6 +18,7 @@ import (
 
 	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
+	dockeroptions "github.com/dokku/dokku/plugins/docker-options"
 	"github.com/dokku/dokku/plugins/logs"
 	nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts"
 	resty "github.com/go-resty/resty/v2"
@@ -1489,6 +1490,37 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
+func getSecurityContext(appName string, phase string) (SecurityContext, error) {
+	securityContext := SecurityContext{}
+	deployOptions, err := dockeroptions.GetSpecifiedDockerOptionsForPhase(appName, phase, []string{
+		"--cap-add",
+		"--cap-drop",
+		"--privileged",
+	})
+	if err != nil {
+		return SecurityContext{}, fmt.Errorf("Error getting deploy options: %w", err)
+	}
+
+	if _, ok := deployOptions["--privileged"]; ok {
+		securityContext.Privileged = true
+	}
+	if capAdd, ok := deployOptions["--cap-add"]; ok {
+		capabilities := []string{}
+		for _, cap := range capAdd {
+			capabilities = append(capabilities, strings.ToUpper(cap))
+		}
+		securityContext.Capabilities.Add = capabilities
+	}
+	if capDrop, ok := deployOptions["--cap-drop"]; ok {
+		capabilities := []string{}
+		for _, cap := range capDrop {
+			capabilities = append(capabilities, strings.ToUpper(cap))
+		}
+		securityContext.Capabilities.Drop = capabilities
+	}
+	return securityContext, nil
+}
+
 func getProcessSpecificKustomizeRootPath(appName string) string {
 	if !hasKustomizeDirectory(appName) {
 		return ""

plugins/scheduler-k3s/template.go

@@ -3,6 +3,7 @@ package scheduler_k3s
 import (
 	"crypto/rand"
 	"fmt"
+	"maps"
 	"os"
 	"strings"
 
@@ -41,15 +42,16 @@ type AppValues struct {
 }
 
 type GlobalValues struct {
-	Annotations  ProcessAnnotations `yaml:"annotations,omitempty"`
+	Annotations     ProcessAnnotations `yaml:"annotations,omitempty"`
-	AppName      string             `yaml:"app_name"`
+	AppName         string             `yaml:"app_name"`
-	DeploymentID string             `yaml:"deployment_id"`
+	DeploymentID    string             `yaml:"deployment_id"`
-	Image        GlobalImage        `yaml:"image"`
+	Image           GlobalImage        `yaml:"image"`
-	Labels       ProcessLabels      `yaml:"labels,omitempty"`
+	Labels          ProcessLabels      `yaml:"labels,omitempty"`
-	Keda         GlobalKedaValues   `yaml:"keda"`
+	Keda            GlobalKedaValues   `yaml:"keda"`
-	Namespace    string             `yaml:"namespace"`
+	Namespace       string             `yaml:"namespace"`
-	Network      GlobalNetwork      `yaml:"network"`
+	Network         GlobalNetwork      `yaml:"network"`
-	Secrets      map[string]string  `yaml:"secrets,omitempty"`
+	Secrets         map[string]string  `yaml:"secrets,omitempty"`
+	SecurityContext SecurityContext    `yaml:"security_context,omitempty"`
 }
 
 type GlobalImage struct {
@@ -351,11 +353,53 @@ type Job struct {
 	Namespace        string
 	ProcessType      string
 	Schedule         string
+	SecurityContext  SecurityContext
 	Suffix           string
 	RemoveContainer  bool
 	WorkingDir       string
 }
 
+// SecurityContext contains the security context for a process
+type SecurityContext struct {
+	// Capabilities contains the capabilities for a process
+	Capabilities SecurityContextCapabilities `yaml:"capabilities,omitempty"`
+	// Privileged contains the privileged flag for a process
+	Privileged bool `yaml:"privileged,omitempty"`
+}
+
+// ToCoreV1SecurityContext converts the security context to a corev1.SecurityContext
+func (s SecurityContext) ToCoreV1SecurityContext() corev1.SecurityContext {
+	securityContext := corev1.SecurityContext{
+		Capabilities: &corev1.Capabilities{},
+		Privileged:   ptr.To(s.Privileged),
+	}
+
+	if len(s.Capabilities.Add) > 0 {
+		capabilities := make([]corev1.Capability, len(s.Capabilities.Add))
+		for i, cap := range s.Capabilities.Add {
+			capabilities[i] = corev1.Capability(cap)
+		}
+		securityContext.Capabilities.Add = capabilities
+	}
+	if len(s.Capabilities.Drop) > 0 {
+		capabilities := make([]corev1.Capability, len(s.Capabilities.Drop))
+		for i, cap := range s.Capabilities.Drop {
+			capabilities[i] = corev1.Capability(cap)
+		}
+		securityContext.Capabilities.Drop = capabilities
+	}
+
+	return securityContext
+}
+
+// SecurityContextCapabilities contains the capabilities for a process
+type SecurityContextCapabilities struct {
+	// Add contains the add capabilities for a process
+	Add []string `yaml:"add,omitempty"`
+	// Drop contains the drop capabilities for a process
+	Drop []string `yaml:"drop,omitempty"`
+}
+
 func templateKubernetesJob(input Job) (batchv1.Job, error) {
 	labels := map[string]string{
 		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
@@ -368,9 +412,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 		"dokku.com/managed":         "true",
 	}
 
-	for key, value := range input.Labels {
+	maps.Copy(labels, input.Labels)
-		labels[key] = value
-	}
 	secretName := fmt.Sprintf("env-%s.%d", input.AppName, input.DeploymentID)
 
 	env := []corev1.EnvVar{}
@@ -417,6 +459,8 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 		podAnnotations[key] = value
 	}
 
+	securityContext := input.SecurityContext.ToCoreV1SecurityContext()
+
 	job := batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
@@ -453,7 +497,8 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 								Limits:   corev1.ResourceList{},
 								Requests: corev1.ResourceList{},
 							},
-							WorkingDir: input.WorkingDir,
+							SecurityContext: &securityContext,
+							WorkingDir:      input.WorkingDir,
 						},
 					},
 					RestartPolicy:      corev1.RestartPolicyNever,

plugins/scheduler-k3s/templates/chart/cron-job.yaml

@@ -102,6 +102,27 @@ spec:
                 {{- end }}
               {{- end }}
             {{- end }}
+            {{- if hasKey $.Values.global "security_context" }}
+            securityContext:
+              {{- if $.Values.global.security_context.privileged }}
+              privileged: true
+              {{- end }}
+              {{- if hasKey $.Values.global.security_context "capabilities" }}
+              capabilities:
+                {{- if hasKey $.Values.global.security_context.capabilities "add" }}
+                add:
+                  {{- range $.Values.global.security_context.capabilities.add }}
+                  - {{ . }}
+                  {{- end }}
+                {{- end }}
+                {{- if hasKey $.Values.global.security_context.capabilities "drop" }}
+                drop:
+                  {{- range $.Values.global.security_context.capabilities.drop }}
+                  - {{ . }}
+                  {{- end }}
+                {{- end }}
+              {{- end }}
+            {{- end }}
             {{- if $.Values.global.image.working_dir }}
             workingDir: {{ $.Values.global.image.working_dir }}
             {{- end }}

plugins/scheduler-k3s/templates/chart/deployment.yaml

@@ -117,6 +117,27 @@ spec:
         readinessProbe:
           {{ $config.healthchecks.readiness | toJson | indent 10 }}
         {{- end }}
+        {{- if hasKey $.Values.global "security_context" }}
+        securityContext:
+          {{- if $.Values.global.security_context.privileged }}
+          privileged: true
+          {{- end }}
+          {{- if hasKey $.Values.global.security_context "capabilities" }}
+          capabilities:
+            {{- if hasKey $.Values.global.security_context.capabilities "add" }}
+            add:
+              {{- range $.Values.global.security_context.capabilities.add }}
+              - {{ . }}
+              {{- end }}
+            {{- end }}
+            {{- if hasKey $.Values.global.security_context.capabilities "drop" }}
+            drop:
+              {{- range $.Values.global.security_context.capabilities.drop }}
+              - {{ . }}
+              {{- end }}
+            {{- end }}
+          {{- end }}
+        {{- end }}
         {{- if $.Values.global.image.working_dir }}
         workingDir: {{ $.Values.global.image.working_dir }}
         {{- end }}

plugins/scheduler-k3s/triggers.go

@@ -387,6 +387,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error getting keda values: %w", err)
 	}
 
+	securityContext, err := getSecurityContext(appName, "deploy")
+	if err != nil {
+		return fmt.Errorf("Error getting security context: %w", err)
+	}
+
 	values := &AppValues{
 		Global: GlobalValues{
 			Annotations:  globalAnnotations,
@@ -407,7 +412,8 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 				PrimaryPort:        primaryPort,
 				PrimaryServicePort: primaryServicePort,
 			},
-			Secrets: map[string]string{},
+			Secrets:         map[string]string{},
+			SecurityContext: securityContext,
 		},
 		Processes: map[string]ProcessValues{},
 	}
@@ -1228,6 +1234,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		}
 	}
 
+	securityContext, err := getSecurityContext(appName, "run")
+	if err != nil {
+		return fmt.Errorf("Error getting security context: %w", err)
+	}
+
 	workingDir := common.GetWorkingDir(appName, image)
 	job, err := templateKubernetesJob(Job{
 		AppName:          appName,
@@ -1243,6 +1254,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		Namespace:        namespace,
 		ProcessType:      processType,
 		RemoveContainer:  rmContainer,
+		SecurityContext:  securityContext,
 		WorkingDir:       workingDir,
 	})
 	if err != nil {

tests/unit/scheduler-k3s-3.bats

@@ -11,12 +11,69 @@ setup() {
   export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
 }
 
-teardown_() {
+teardown() {
   global_teardown
   dokku nginx:start
   uninstall_k3s || true
 }
 
+@test "(scheduler-k3s) security context" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "null"
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.privileged'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "null"
+
+  run /bin/bash -c "dokku docker-options:add $TEST_APP deploy --cap-add=NET_ADMIN"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku docker-options:add $TEST_APP deploy --privileged"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:restart $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "NET_ADMIN"
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.privileged'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "true"
+}
+
 @test "(scheduler-k3s) kustomize" {
   if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
     skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"