feat: manage node-level kernel sysctls on the k3s scheduler

Sysctls the kernel does not namespace, such as `vm.max_map_count`, cannot be set from a pod spec and previously had no answer beyond editing `/etc/sysctl.d` on each host by hand. `scheduler-k3s:node-sysctls:set` now applies them through a privileged daemonset, which reaches nodes joined later and reapplies after a reboot. Sysctls may be scoped to a node profile, with a profile scope inheriting the global values and overriding them on conflict so that every node is covered by exactly one daemonset. Clearing a sysctl stops dokku managing it but does not restore the previous value, which persists until the node reboots.
2026-08-07 02:46:30 -04:00
parent cd1089500b
commit 44cd566178
12 changed files with 964 additions and 20 deletions
--- a/plugins/scheduler-k3s/node_sysctls.go
+++ b/plugins/scheduler-k3s/node_sysctls.go
@@ -0,0 +1,324 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// NodeSysctlsValues contains the values for a dokku-managed node sysctls helm chart
+type NodeSysctlsValues struct {
+	Global NodeSysctlsGlobalValues `yaml:"global"`
+}
+
+// NodeSysctlsGlobalValues contains the global values for the node sysctls chart
+type NodeSysctlsGlobalValues struct {
+	Image       string   `yaml:"image"`
+	PauseImage  string   `yaml:"pause_image"`
+	ProfileName string   `yaml:"profile_name,omitempty"`
+	ReleaseName string   `yaml:"release_name"`
+	Sysctls     []Sysctl `yaml:"sysctls"`
+}
+
+// nodeSysctlScope is a resolved set of sysctls destined for a single DaemonSet
+type nodeSysctlScope struct {
+	// ProfileName is the node profile this scope targets, empty for the global scope
+	ProfileName string
+	// ReleaseName is the helm release backing this scope
+	ReleaseName string
+	// Sysctls are the fully resolved sysctls for this scope
+	Sysctls []Sysctl
+}
+
+// getNodeSysctlsProperty returns the property name backing a node sysctls scope
+func getNodeSysctlsProperty(profileName string) string {
+	if profileName == "" {
+		return "node-sysctls.global"
+	}
+
+	return fmt.Sprintf("node-sysctls.profile.%s", profileName)
+}
+
+// getNodeSysctlsReleaseName returns the helm release name for a node sysctls scope
+func getNodeSysctlsReleaseName(profileName string) string {
+	if profileName == "" {
+		return "dokku-node-sysctls-global"
+	}
+
+	return fmt.Sprintf("dokku-node-sysctls-profile-%s", profileName)
+}
+
+// getNodeSysctls returns the sysctls stored against a single scope
+func getNodeSysctls(profileName string) (map[string]string, error) {
+	sysctls, err := common.PropertyMapGet("scheduler-k3s", "--global", getNodeSysctlsProperty(profileName))
+	if err != nil {
+		return nil, fmt.Errorf("Unable to read node sysctls: %w", err)
+	}
+
+	return sysctls, nil
+}
+
+// listNodeProfileNames returns the names of every stored node profile
+func listNodeProfileNames() ([]string, error) {
+	properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "node-profile-")
+	if err != nil {
+		return nil, fmt.Errorf("Unable to get node profiles: %w", err)
+	}
+
+	names := []string{}
+	for property, data := range properties {
+		if !strings.HasSuffix(property, ".json") {
+			continue
+		}
+
+		var profile NodeProfile
+		if err := json.Unmarshal([]byte(data), &profile); err != nil {
+			return nil, fmt.Errorf("Unable to unmarshal node profile: %w", err)
+		}
+
+		names = append(names, profile.Name)
+	}
+
+	sort.Strings(names)
+	return names, nil
+}
+
+// sortedSysctls converts a name/value map into sysctls sorted by name so the
+// rendered manifest is stable across runs
+func sortedSysctls(values map[string]string) []Sysctl {
+	sysctls := make([]Sysctl, 0, len(values))
+	for name, value := range values {
+		sysctls = append(sysctls, Sysctl{Name: name, Value: value})
+	}
+
+	sort.Slice(sysctls, func(i int, j int) bool {
+		return sysctls[i].Name < sysctls[j].Name
+	})
+
+	return sysctls
+}
+
+// mergeNodeSysctls layers a profile's sysctls over the global ones, with the profile
+// winning on conflict, and returns the result sorted by name. Profile scopes must
+// carry the global values too: profiled nodes are excluded from the global DaemonSet,
+// so anything omitted here would never reach them.
+func mergeNodeSysctls(global map[string]string, profile map[string]string) []Sysctl {
+	merged := map[string]string{}
+	for name, value := range global {
+		merged[name] = value
+	}
+	for name, value := range profile {
+		merged[name] = value
+	}
+
+	return sortedSysctls(merged)
+}
+
+// resolveNodeSysctlScopes returns one scope per DaemonSet that should exist, with
+// profile scopes carrying the global sysctls merged underneath their own. Every
+// node matches exactly one scope: profiled nodes match their profile's DaemonSet,
+// and unprofiled nodes match the global one, so no two DaemonSets ever write the
+// same sysctl on the same node.
+func resolveNodeSysctlScopes() ([]nodeSysctlScope, error) {
+	globalSysctls, err := getNodeSysctls("")
+	if err != nil {
+		return nil, err
+	}
+
+	scopes := []nodeSysctlScope{
+		{
+			ReleaseName: getNodeSysctlsReleaseName(""),
+			Sysctls:     sortedSysctls(globalSysctls),
+		},
+	}
+
+	profileNames, err := listNodeProfileNames()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, profileName := range profileNames {
+		profileSysctls, err := getNodeSysctls(profileName)
+		if err != nil {
+			return nil, err
+		}
+
+		scopes = append(scopes, nodeSysctlScope{
+			ProfileName: profileName,
+			ReleaseName: getNodeSysctlsReleaseName(profileName),
+			Sysctls:     mergeNodeSysctls(globalSysctls, profileSysctls),
+		})
+	}
+
+	return scopes, nil
+}
+
+// CreateOrUpdateNodeSysctls reconciles every node sysctls DaemonSet against the
+// stored properties. All scopes are reconciled on every call rather than only the
+// mutated one, since profile scopes inherit the global sysctls at render time and
+// would otherwise serve a stale value after a global change.
+func CreateOrUpdateNodeSysctls(ctx context.Context) error {
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping node sysctls sync")
+		return nil
+	}
+
+	scopes, err := resolveNodeSysctlScopes()
+	if err != nil {
+		return err
+	}
+
+	for _, scope := range scopes {
+		if len(scope.Sysctls) == 0 {
+			if err := deleteNodeSysctlsRelease(scope.ReleaseName); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if err := installNodeSysctlsChart(ctx, scope); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// installNodeSysctlsChart installs or upgrades the DaemonSet backing a single scope
+func installNodeSysctlsChart(ctx context.Context, scope nodeSysctlScope) error {
+	chartDir, err := os.MkdirTemp("", "dokku-node-sysctls-chart-")
+	if err != nil {
+		return fmt.Errorf("error creating chart directory: %w", err)
+	}
+	defer os.RemoveAll(chartDir)
+
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
+		return fmt.Errorf("error creating chart templates directory: %w", err)
+	}
+
+	chart := &Chart{
+		ApiVersion: "v2",
+		AppVersion: "1.0.0",
+		Name:       scope.ReleaseName,
+		Icon:       "https://dokku.com/assets/dokku-logo.svg",
+		Version:    "0.0.1",
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: chart,
+		Path:   filepath.Join(chartDir, "Chart.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing chart: %w", err)
+	}
+
+	b, err := templates.ReadFile("templates/node-sysctls-chart/templates/daemonset.yaml")
+	if err != nil {
+		return fmt.Errorf("error reading node-sysctls template: %w", err)
+	}
+
+	filename := filepath.Join(chartDir, "templates", "daemonset.yaml")
+	if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
+		return fmt.Errorf("error writing node-sysctls template: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(filename)
+	}
+
+	values := &NodeSysctlsValues{
+		Global: NodeSysctlsGlobalValues{
+			Image:       getComputedNodeSysctlsImage(),
+			PauseImage:  getComputedNodeSysctlsPauseImage(),
+			ProfileName: scope.ProfileName,
+			ReleaseName: scope.ReleaseName,
+			Sysctls:     scope.Sysctls,
+		},
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: values,
+		Path:   filepath.Join(chartDir, "values.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing values: %w", err)
+	}
+
+	helmAgent, err := NewHelmAgent(NodeSysctlsNamespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	chartPath, err := filepath.Abs(chartDir)
+	if err != nil {
+		return fmt.Errorf("error getting chart path: %w", err)
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Applying node sysctls for %s", nodeSysctlsScopeLabel(scope.ProfileName)))
+	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
+		ChartPath:   chartPath,
+		Namespace:   NodeSysctlsNamespace,
+		ReleaseName: scope.ReleaseName,
+		Wait:        false,
+	})
+	if err != nil {
+		return fmt.Errorf("error installing node sysctls chart: %w", err)
+	}
+
+	return nil
+}
+
+// deleteNodeSysctlsRelease removes a node sysctls DaemonSet when its scope resolves
+// to an empty set. The kernel values it wrote are not reverted; they persist on the
+// affected nodes until those nodes reboot.
+func deleteNodeSysctlsRelease(releaseName string) error {
+	helmAgent, err := NewHelmAgent(NodeSysctlsNamespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	exists, err := helmAgent.ChartExists(releaseName)
+	if err != nil {
+		return fmt.Errorf("error checking if node sysctls chart exists: %w", err)
+	}
+
+	if !exists {
+		return nil
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Removing node sysctls release %s", releaseName))
+	if err := helmAgent.UninstallChart(releaseName); err != nil {
+		return fmt.Errorf("error uninstalling node sysctls chart: %w", err)
+	}
+
+	return nil
+}
+
+// DeleteNodeSysctls removes the stored sysctls and DaemonSet for a single node profile
+func DeleteNodeSysctls(ctx context.Context, profileName string) error {
+	if err := common.PropertyDelete("scheduler-k3s", "--global", getNodeSysctlsProperty(profileName)); err != nil {
+		return fmt.Errorf("Unable to delete node sysctls: %w", err)
+	}
+
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping node sysctls deletion")
+		return nil
+	}
+
+	return deleteNodeSysctlsRelease(getNodeSysctlsReleaseName(profileName))
+}
+
+// nodeSysctlsScopeLabel returns a human readable name for a node sysctls scope
+func nodeSysctlsScopeLabel(profileName string) string {
+	if profileName == "" {
+		return "unprofiled nodes"
+	}
+
+	return fmt.Sprintf("node profile %s", profileName)
+}