From 551c6b1c436cb48c4f2c3c1c4ed10005de76a544 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 23 Sep 2026 02:09:33 -0400 Subject: [PATCH] feat: report each node-sysctls scope's stored map `scheduler-k3s:node-sysctls:report` renders each scope's resolved set, so a profile's entry carries every globally-set sysctl underneath its own and there is no way to read back the map the profile itself stores. Since `node-sysctls:set --replace --profile` and `node-sysctls:clear --profile` write only that map, a tool converging a profile against a declared map sees permanent drift. The new `--stored` flag reports the map each scope owns, and `--global` and `--profile` narrow the report to a single scope. The default output is unchanged. --- docs/deployment/schedulers/k3s.md | 39 +++++- plugins/scheduler-k3s/node_sysctls.go | 58 ++++++++- plugins/scheduler-k3s/node_sysctls_test.go | 119 ++++++++++++++++++ .../scheduler-k3s/src/commands/commands.go | 2 +- .../src/subcommands/subcommands.go | 15 ++- plugins/scheduler-k3s/subcommands.go | 48 ++++--- tests/unit/scheduler-k3s-node-sysctls-2.bats | 91 ++++++++++++++ 7 files changed, 344 insertions(+), 28 deletions(-) create mode 100644 tests/unit/scheduler-k3s-node-sysctls-2.bats diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index b9aae61a0..74545ab8c 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -25,7 +25,7 @@ scheduler-k3s:labels:report [|--global] [--format stdout|json] [--process-t scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE] # Clear all node-level kernel sysctls for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:set () [--global|--profile PROFILE] # Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:set --replace [ ...] [--global|--profile PROFILE] # Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile -scheduler-k3s:node-sysctls:report [--format stdout|json] # Displays the node-level kernel sysctls applied to each scope +scheduler-k3s:node-sysctls:report [--format stdout|json] [--stored] [--global|--profile PROFILE] # Displays the node-level kernel sysctls for each scope scheduler-k3s:preview [--context N] [--show-secrets] [--show-secrets-decoded] # Displays a diff between the current and next deployment for an app scheduler-k3s:profiles:add [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] # Adds a node profile to the k3s cluster scheduler-k3s:profiles:list [--format json|stdout] # Lists all node profiles in the k3s cluster @@ -1008,16 +1008,51 @@ dokku scheduler-k3s:node-sysctls:clear --profile edge-workers Clearing a scope removes its DaemonSet. As with clearing a single sysctl, the values it last wrote stay in place on affected nodes until they reboot. -Use `node-sysctls:report` to see the resolved set for every scope. +Use `node-sysctls:report` to see the resolved set every scope's DaemonSet applies. A profile's entry therefore includes everything it inherits from the global scope. ```shell dokku scheduler-k3s:node-sysctls:report ``` +``` +scope sysctl value +--global vm.overcommit_memory 1 +--global vm.swappiness 20 +edge-workers vm.overcommit_memory 1 +edge-workers vm.swappiness 60 +``` + ```shell dokku scheduler-k3s:node-sysctls:report --format json ``` +Because that report resolves each scope, a profile's entry does not distinguish a sysctl the profile sets itself from one it inherits. To report only the map each scope stores - the map its own `node-sysctls:set` and `node-sysctls:clear` calls write - add the `--stored` flag: + +```shell +dokku scheduler-k3s:node-sysctls:report --stored +``` + +``` +scope sysctl value +--global vm.overcommit_memory 1 +--global vm.swappiness 20 +edge-workers vm.swappiness 60 +``` + +The global scope stores exactly what it applies, so `--stored` reports the same thing for `--global` either way. + +A configuration tool holding a declared map for a scope should compare it against `--stored`. Comparing against the resolved set never converges a profile, since a profile that stores nothing of its own still reports every globally-set sysctl. + +The report covers every scope unless `--global` or `--profile` narrows it to one, the same way those flags scope `node-sysctls:set` and `node-sysctls:clear`. Only one of the two may be given. + +```shell +dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers --format json +``` + +```json +{"edge-workers":{"vm.swappiness":"60"}} +``` + The DaemonSet pulls `busybox` and `registry.k8s.io/pause` by default. On an air-gapped cluster or one behind a registry mirror, point them elsewhere: ```shell diff --git a/plugins/scheduler-k3s/node_sysctls.go b/plugins/scheduler-k3s/node_sysctls.go index e66c73040..7fdc1d6d5 100644 --- a/plugins/scheduler-k3s/node_sysctls.go +++ b/plugins/scheduler-k3s/node_sysctls.go @@ -35,6 +35,9 @@ type nodeSysctlScope struct { ReleaseName string // Sysctls are the fully resolved sysctls for this scope Sysctls []Sysctl + // StoredSysctls are only the sysctls this scope stores itself, with nothing + // inherited from the global scope + StoredSysctls []Sysctl } // getNodeSysctlsProperty returns the property name backing a node sysctls scope @@ -130,17 +133,20 @@ func mergeNodeSysctls(global map[string]string, profile map[string]string) []Sys // profile scopes carrying the global sysctls merged underneath their own. Every // node matches exactly one scope: profiled nodes match their profile's DaemonSet, // and unprofiled nodes match the global one, so no two DaemonSets ever write the -// same sysctl on the same node. +// same sysctl on the same node. Each scope also carries the map it stores on its +// own, which is what that scope's set and clear commands write. func resolveNodeSysctlScopes() ([]nodeSysctlScope, error) { globalSysctls, err := getNodeSysctls("") if err != nil { return nil, err } + global := sortedSysctls(globalSysctls) scopes := []nodeSysctlScope{ { - ReleaseName: getNodeSysctlsReleaseName(""), - Sysctls: sortedSysctls(globalSysctls), + ReleaseName: getNodeSysctlsReleaseName(""), + Sysctls: global, + StoredSysctls: global, }, } @@ -156,15 +162,45 @@ func resolveNodeSysctlScopes() ([]nodeSysctlScope, error) { } scopes = append(scopes, nodeSysctlScope{ - ProfileName: profileName, - ReleaseName: getNodeSysctlsReleaseName(profileName), - Sysctls: mergeNodeSysctls(globalSysctls, profileSysctls), + ProfileName: profileName, + ReleaseName: getNodeSysctlsReleaseName(profileName), + Sysctls: mergeNodeSysctls(globalSysctls, profileSysctls), + StoredSysctls: sortedSysctls(profileSysctls), }) } return scopes, nil } +// reportSysctls returns the sysctls a report should render for this scope: the map +// the scope stores on its own when stored is set, and otherwise the resolved set its +// DaemonSet applies. +func (scope nodeSysctlScope) reportSysctls(stored bool) []Sysctl { + if stored { + return scope.StoredSysctls + } + + return scope.Sysctls +} + +// filterNodeSysctlScopes narrows resolved scopes to the single scope a report asked +// for, with global selecting the scope no profile owns. A report naming neither gets +// every scope. +func filterNodeSysctlScopes(scopes []nodeSysctlScope, global bool, profileName string) []nodeSysctlScope { + if !global && profileName == "" { + return scopes + } + + filtered := []nodeSysctlScope{} + for _, scope := range scopes { + if scope.ProfileName == profileName { + filtered = append(filtered, scope) + } + } + + return filtered +} + // CreateOrUpdateNodeSysctls reconciles every node sysctls DaemonSet against the // stored properties. All scopes are reconciled on every call rather than only the // mutated one, since profile scopes inherit the global sysctls at render time and @@ -330,6 +366,16 @@ func DeleteNodeSysctls(ctx context.Context, profileName string) error { return deleteNodeSysctlsRelease(getNodeSysctlsReleaseName(profileName)) } +// nodeSysctlsScopeKey returns the key a report addresses a node sysctls scope by, +// matching the flag that scope is set and cleared with +func nodeSysctlsScopeKey(profileName string) string { + if profileName == "" { + return "--global" + } + + return profileName +} + // nodeSysctlsScopeLabel returns a human readable name for a node sysctls scope func nodeSysctlsScopeLabel(profileName string) string { if profileName == "" { diff --git a/plugins/scheduler-k3s/node_sysctls_test.go b/plugins/scheduler-k3s/node_sysctls_test.go index 00f2e54da..38960f113 100644 --- a/plugins/scheduler-k3s/node_sysctls_test.go +++ b/plugins/scheduler-k3s/node_sysctls_test.go @@ -303,3 +303,122 @@ func TestMergeNodeSysctlsDoesNotMutateInputs(t *testing.T) { t.Errorf("mergeNodeSysctls() mutated the profile map: %v", profile) } } + +func TestNodeSysctlsScopeKey(t *testing.T) { + if got := nodeSysctlsScopeKey(""); got != "--global" { + t.Errorf("nodeSysctlsScopeKey(\"\") = %q, want --global", got) + } + if got := nodeSysctlsScopeKey("edge-workers"); got != "edge-workers" { + t.Errorf("nodeSysctlsScopeKey(\"edge-workers\") = %q, want edge-workers", got) + } +} + +// TestNodeSysctlScopeReportSysctls asserts a report can read back the map a scope +// stores. A profile storing nothing of its own still resolves to every global sysctl, +// so the resolved set alone cannot tell a stored value from an inherited one. +func TestNodeSysctlScopeReportSysctls(t *testing.T) { + global := []Sysctl{{Name: "vm.max_map_count", Value: "262144"}} + profile := nodeSysctlScope{ + ProfileName: "edge-workers", + Sysctls: []Sysctl{ + {Name: "vm.max_map_count", Value: "262144"}, + {Name: "vm.swappiness", Value: "10"}, + }, + StoredSysctls: []Sysctl{{Name: "vm.swappiness", Value: "10"}}, + } + + cases := []struct { + name string + scope nodeSysctlScope + stored bool + want []Sysctl + }{ + { + name: "profile resolved set includes inherited sysctls", + scope: profile, + stored: false, + want: profile.Sysctls, + }, + { + name: "profile stored map excludes inherited sysctls", + scope: profile, + stored: true, + want: profile.StoredSysctls, + }, + { + name: "global scope resolves to what it stores", + scope: nodeSysctlScope{Sysctls: global, StoredSysctls: global}, + stored: true, + want: global, + }, + { + name: "profile storing nothing reports an empty map", + scope: nodeSysctlScope{ProfileName: "edge-workers", Sysctls: global, StoredSysctls: []Sysctl{}}, + stored: true, + want: []Sysctl{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := tc.scope.reportSysctls(tc.stored) + if len(got) != len(tc.want) { + t.Fatalf("reportSysctls(%t) = %v, want %v", tc.stored, got, tc.want) + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("reportSysctls(%t)[%d] = %v, want %v", tc.stored, i, got[i], tc.want[i]) + } + } + }) + } +} + +func TestFilterNodeSysctlScopes(t *testing.T) { + scopes := []nodeSysctlScope{ + {ReleaseName: "dokku-node-sysctls-global"}, + {ProfileName: "edge-workers", ReleaseName: "dokku-node-sysctls-profile-edge-workers"}, + {ProfileName: "gpu-workers", ReleaseName: "dokku-node-sysctls-profile-gpu-workers"}, + } + + cases := []struct { + name string + global bool + profileName string + want []string + }{ + { + name: "no filter keeps every scope", + want: []string{"", "edge-workers", "gpu-workers"}, + }, + { + name: "global keeps only the unprofiled scope", + global: true, + want: []string{""}, + }, + { + name: "profile keeps only its own scope", + profileName: "edge-workers", + want: []string{"edge-workers"}, + }, + { + name: "unknown profile keeps nothing", + profileName: "missing-profile", + want: []string{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := filterNodeSysctlScopes(scopes, tc.global, tc.profileName) + if len(got) != len(tc.want) { + t.Fatalf("filterNodeSysctlScopes() returned %d scopes, want %d", len(got), len(tc.want)) + } + for i := range got { + if got[i].ProfileName != tc.want[i] { + t.Errorf("filterNodeSysctlScopes()[%d].ProfileName = %q, want %q", i, got[i].ProfileName, tc.want[i]) + } + } + }) + } +} diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index e593aad45..fa22ac357 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -38,7 +38,7 @@ Additional commands:` scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE], Clear all node-level kernel sysctls for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:set () [--global|--profile PROFILE], Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:set --replace [ ...] [--global|--profile PROFILE], Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile - scheduler-k3s:node-sysctls:report [--format stdout|json], Displays the node-level kernel sysctls applied to each scope + scheduler-k3s:node-sysctls:report [--format stdout|json] [--stored] [--global|--profile PROFILE], Displays the node-level kernel sysctls for each scope scheduler-k3s:preview [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app scheduler-k3s:profiles:add [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster scheduler-k3s:profiles:list [--format json|stdout], Lists all node profiles in the k3s cluster diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 5a7e07dde..65d117521 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -257,8 +257,21 @@ func main() { case "node-sysctls:report": args := flag.NewFlagSet("scheduler-k3s:node-sysctls:report", flag.ExitOnError) format := args.String("format", "stdout", "format: [ stdout | json ]") + global := args.Bool("global", false, "--global: report only on all nodes without a node profile") + profileName := args.String("profile", "", "--profile: report only on a node profile instead of every scope") + stored := args.Bool("stored", false, "--stored: report the sysctls each scope stores itself rather than the resolved set it applies") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandNodeSysctlsReport(*format) + if *global && *profileName != "" { + err = fmt.Errorf("Only one of --global and --profile may be specified") + break + } + + err = scheduler_k3s.CommandNodeSysctlsReport(scheduler_k3s.NodeSysctlsReportInput{ + Format: *format, + Global: *global, + ProfileName: *profileName, + Stored: *stored, + }) case "preview": args := flag.NewFlagSet("scheduler-k3s:preview", flag.ExitOnError) context := args.Int("context", 3, "--context: number of unchanged lines of context around each change (-1 for full output)") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 8ce382dcf..4edecbec9 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -152,10 +152,29 @@ func warnUnmanagedSysctls(names []string) { } } -// CommandNodeSysctlsReport displays the configured node-level kernel sysctls -func CommandNodeSysctlsReport(format string) error { - if format != "stdout" && format != "json" { - return fmt.Errorf("Invalid format: %s", format) +// NodeSysctlsReportInput captures the inputs accepted by node-sysctls:report. +// Global and ProfileName are mutually exclusive, and naming neither reports every +// scope rather than the global one. +type NodeSysctlsReportInput struct { + Format string + Global bool + ProfileName string + Stored bool +} + +// CommandNodeSysctlsReport displays the configured node-level kernel sysctls. A named +// profile only has to exist to be reported on, unlike one being written to: a profile +// whose legacy name cannot back a helm release still resolves to a scope, and the map +// it stores is worth reading back even though nothing applies it. +func CommandNodeSysctlsReport(input NodeSysctlsReportInput) error { + if input.Format != "stdout" && input.Format != "json" { + return fmt.Errorf("Invalid format: %s", input.Format) + } + + if input.ProfileName != "" { + if err := verifyNodeProfileExists(input.ProfileName); err != nil { + return err + } } scopes, err := resolveNodeSysctlScopes() @@ -163,19 +182,16 @@ func CommandNodeSysctlsReport(format string) error { return err } - if format == "json" { + scopes = filterNodeSysctlScopes(scopes, input.Global, input.ProfileName) + + if input.Format == "json" { output := map[string]map[string]string{} for _, scope := range scopes { - key := scope.ProfileName - if key == "" { - key = "--global" - } - entries := map[string]string{} - for _, sysctl := range scope.Sysctls { + for _, sysctl := range scope.reportSysctls(input.Stored) { entries[sysctl.Name] = sysctl.Value } - output[key] = entries + output[nodeSysctlsScopeKey(scope.ProfileName)] = entries } b, err := json.Marshal(output) @@ -189,12 +205,8 @@ func CommandNodeSysctlsReport(format string) error { lines := []string{"scope|sysctl|value"} for _, scope := range scopes { - scopeName := scope.ProfileName - if scopeName == "" { - scopeName = "--global" - } - - for _, sysctl := range scope.Sysctls { + scopeName := nodeSysctlsScopeKey(scope.ProfileName) + for _, sysctl := range scope.reportSysctls(input.Stored) { lines = append(lines, fmt.Sprintf("%s|%s|%s", scopeName, sysctl.Name, sysctl.Value)) } } diff --git a/tests/unit/scheduler-k3s-node-sysctls-2.bats b/tests/unit/scheduler-k3s-node-sysctls-2.bats new file mode 100644 index 000000000..fd2e745d6 --- /dev/null +++ b/tests/unit/scheduler-k3s-node-sysctls-2.bats @@ -0,0 +1,91 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup +} + +teardown() { + dokku scheduler-k3s:node-sysctls:clear --global >/dev/null 2>/dev/null || true + dokku scheduler-k3s:node-sysctls:clear --profile edge-workers >/dev/null 2>/dev/null || true + dokku scheduler-k3s:profiles:remove edge-workers >/dev/null 2>/dev/null || true + global_teardown +} + +@test "(scheduler-k3s:node-sysctls:report) --stored returns only the map each scope owns" { + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.overcommit_memory=1 vm.swappiness=20" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:profiles:add edge-workers --role worker" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --profile edge-workers vm.swappiness=60" + assert_success + + # the resolved set a profile's daemonset applies carries the global sysctls underneath its own + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"edge-workers\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'" + assert_success + assert_output "vm.overcommit_memory=1,vm.swappiness=60" + + # the stored map holds only what the profile itself was set to + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --format json | jq -r '.\"edge-workers\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'" + assert_success + assert_output "vm.swappiness=60" + + # the global scope stores exactly what it applies + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --format json | jq -r '.\"--global\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'" + assert_success + assert_output "vm.overcommit_memory=1,vm.swappiness=20" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --global --format json | jq -rS '.'" + assert_success + stored_global="$output" + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --global --format json | jq -rS '.'" + assert_success + assert_output "$stored_global" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers --format json | jq -rS 'keys | join(\",\")'" + assert_success + assert_output "edge-workers" + + # the stdout renderer honours both flags too + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers" + assert_success + assert_output_contains "edge-workers" + assert_output_not_contains "vm.overcommit_memory" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --profile edge-workers" + assert_success + + # a cleared profile keeps inheriting the global sysctls, so only the stored map converges + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"edge-workers\" | length'" + assert_success + assert_output "2" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers --format json | jq -r '.\"edge-workers\" | length'" + assert_success + assert_output "0" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers" + assert_success + assert_output_not_contains "vm.overcommit_memory" + assert_output_not_contains "vm.swappiness" +} + +@test "(scheduler-k3s:node-sysctls:report) rejects conflicting and unknown scopes" { + run /bin/bash -c "dokku scheduler-k3s:profiles:add edge-workers --role worker" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --global --profile edge-workers" + assert_failure + assert_output_contains "Only one of --global and --profile may be specified" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --profile missing-profile" + assert_failure + assert_output_contains "Node profile missing-profile not found" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --format bogus" + assert_failure + assert_output_contains "Invalid format: bogus" +}