Merge pull request #9075 from dokku/9073-expose-each-node-sysctls-scope-s-stored-map-in-the-report

Report each node-sysctls scope's stored map

docs/deployment/schedulers/k3s.md

@@ -25,7 +25,7 @@ scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-t
 scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE] # Clear all node-level kernel sysctls for unprofiled nodes or a single node profile
 scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE] # Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
 scheduler-k3s:node-sysctls:set --replace <sysctl=value> [<sysctl=value> ...] [--global|--profile PROFILE] # Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile
-scheduler-k3s:node-sysctls:report [--format stdout|json] # Displays the node-level kernel sysctls applied to each scope
+scheduler-k3s:node-sysctls:report [--format stdout|json] [--stored] [--global|--profile PROFILE] # Displays the node-level kernel sysctls for each scope
 scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded] # Displays a diff between the current and next deployment for an app
 scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] # Adds a node profile to the k3s cluster
 scheduler-k3s:profiles:list [--format json|stdout] # Lists all node profiles in the k3s cluster
@@ -1008,16 +1008,51 @@ dokku scheduler-k3s:node-sysctls:clear --profile edge-workers
 
 Clearing a scope removes its DaemonSet. As with clearing a single sysctl, the values it last wrote stay in place on affected nodes until they reboot.
 
-Use `node-sysctls:report` to see the resolved set for every scope.
+Use `node-sysctls:report` to see the resolved set every scope's DaemonSet applies. A profile's entry therefore includes everything it inherits from the global scope.
 
 ```shell
 dokku scheduler-k3s:node-sysctls:report
 ```
 
+```
+scope         sysctl                value
+--global      vm.overcommit_memory  1
+--global      vm.swappiness         20
+edge-workers  vm.overcommit_memory  1
+edge-workers  vm.swappiness         60
+```
+
 ```shell
 dokku scheduler-k3s:node-sysctls:report --format json
 ```
 
+Because that report resolves each scope, a profile's entry does not distinguish a sysctl the profile sets itself from one it inherits. To report only the map each scope stores - the map its own `node-sysctls:set` and `node-sysctls:clear` calls write - add the `--stored` flag:
+
+```shell
+dokku scheduler-k3s:node-sysctls:report --stored
+```
+
+```
+scope         sysctl                value
+--global      vm.overcommit_memory  1
+--global      vm.swappiness         20
+edge-workers  vm.swappiness         60
+```
+
+The global scope stores exactly what it applies, so `--stored` reports the same thing for `--global` either way.
+
+A configuration tool holding a declared map for a scope should compare it against `--stored`. Comparing against the resolved set never converges a profile, since a profile that stores nothing of its own still reports every globally-set sysctl.
+
+The report covers every scope unless `--global` or `--profile` narrows it to one, the same way those flags scope `node-sysctls:set` and `node-sysctls:clear`. Only one of the two may be given.
+
+```shell
+dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers --format json
+```
+
+```json
+{"edge-workers":{"vm.swappiness":"60"}}
+```
+
 The DaemonSet pulls `busybox` and `registry.k8s.io/pause` by default. On an air-gapped cluster or one behind a registry mirror, point them elsewhere:
 
 ```shell

plugins/scheduler-k3s/node_sysctls.go

@@ -35,6 +35,9 @@ type nodeSysctlScope struct {
 	ReleaseName string
 	// Sysctls are the fully resolved sysctls for this scope
 	Sysctls []Sysctl
+	// StoredSysctls are only the sysctls this scope stores itself, with nothing
+	// inherited from the global scope
+	StoredSysctls []Sysctl
 }
 
 // getNodeSysctlsProperty returns the property name backing a node sysctls scope
@@ -130,17 +133,20 @@ func mergeNodeSysctls(global map[string]string, profile map[string]string) []Sys
 // profile scopes carrying the global sysctls merged underneath their own. Every
 // node matches exactly one scope: profiled nodes match their profile's DaemonSet,
 // and unprofiled nodes match the global one, so no two DaemonSets ever write the
-// same sysctl on the same node.
+// same sysctl on the same node. Each scope also carries the map it stores on its
+// own, which is what that scope's set and clear commands write.
 func resolveNodeSysctlScopes() ([]nodeSysctlScope, error) {
 	globalSysctls, err := getNodeSysctls("")
 	if err != nil {
 		return nil, err
 	}
 
+	global := sortedSysctls(globalSysctls)
 	scopes := []nodeSysctlScope{
 		{
-			ReleaseName: getNodeSysctlsReleaseName(""),
+			ReleaseName:   getNodeSysctlsReleaseName(""),
-			Sysctls:     sortedSysctls(globalSysctls),
+			Sysctls:       global,
+			StoredSysctls: global,
 		},
 	}
 
@@ -156,15 +162,45 @@ func resolveNodeSysctlScopes() ([]nodeSysctlScope, error) {
 		}
 
 		scopes = append(scopes, nodeSysctlScope{
-			ProfileName: profileName,
+			ProfileName:   profileName,
-			ReleaseName: getNodeSysctlsReleaseName(profileName),
+			ReleaseName:   getNodeSysctlsReleaseName(profileName),
-			Sysctls:     mergeNodeSysctls(globalSysctls, profileSysctls),
+			Sysctls:       mergeNodeSysctls(globalSysctls, profileSysctls),
+			StoredSysctls: sortedSysctls(profileSysctls),
 		})
 	}
 
 	return scopes, nil
 }
 
+// reportSysctls returns the sysctls a report should render for this scope: the map
+// the scope stores on its own when stored is set, and otherwise the resolved set its
+// DaemonSet applies.
+func (scope nodeSysctlScope) reportSysctls(stored bool) []Sysctl {
+	if stored {
+		return scope.StoredSysctls
+	}
+
+	return scope.Sysctls
+}
+
+// filterNodeSysctlScopes narrows resolved scopes to the single scope a report asked
+// for, with global selecting the scope no profile owns. A report naming neither gets
+// every scope.
+func filterNodeSysctlScopes(scopes []nodeSysctlScope, global bool, profileName string) []nodeSysctlScope {
+	if !global && profileName == "" {
+		return scopes
+	}
+
+	filtered := []nodeSysctlScope{}
+	for _, scope := range scopes {
+		if scope.ProfileName == profileName {
+			filtered = append(filtered, scope)
+		}
+	}
+
+	return filtered
+}
+
 // CreateOrUpdateNodeSysctls reconciles every node sysctls DaemonSet against the
 // stored properties. All scopes are reconciled on every call rather than only the
 // mutated one, since profile scopes inherit the global sysctls at render time and
@@ -330,6 +366,16 @@ func DeleteNodeSysctls(ctx context.Context, profileName string) error {
 	return deleteNodeSysctlsRelease(getNodeSysctlsReleaseName(profileName))
 }
 
+// nodeSysctlsScopeKey returns the key a report addresses a node sysctls scope by,
+// matching the flag that scope is set and cleared with
+func nodeSysctlsScopeKey(profileName string) string {
+	if profileName == "" {
+		return "--global"
+	}
+
+	return profileName
+}
+
 // nodeSysctlsScopeLabel returns a human readable name for a node sysctls scope
 func nodeSysctlsScopeLabel(profileName string) string {
 	if profileName == "" {

plugins/scheduler-k3s/node_sysctls_test.go

@@ -303,3 +303,122 @@ func TestMergeNodeSysctlsDoesNotMutateInputs(t *testing.T) {
 		t.Errorf("mergeNodeSysctls() mutated the profile map: %v", profile)
 	}
 }
+
+func TestNodeSysctlsScopeKey(t *testing.T) {
+	if got := nodeSysctlsScopeKey(""); got != "--global" {
+		t.Errorf("nodeSysctlsScopeKey(\"\") = %q, want --global", got)
+	}
+	if got := nodeSysctlsScopeKey("edge-workers"); got != "edge-workers" {
+		t.Errorf("nodeSysctlsScopeKey(\"edge-workers\") = %q, want edge-workers", got)
+	}
+}
+
+// TestNodeSysctlScopeReportSysctls asserts a report can read back the map a scope
+// stores. A profile storing nothing of its own still resolves to every global sysctl,
+// so the resolved set alone cannot tell a stored value from an inherited one.
+func TestNodeSysctlScopeReportSysctls(t *testing.T) {
+	global := []Sysctl{{Name: "vm.max_map_count", Value: "262144"}}
+	profile := nodeSysctlScope{
+		ProfileName: "edge-workers",
+		Sysctls: []Sysctl{
+			{Name: "vm.max_map_count", Value: "262144"},
+			{Name: "vm.swappiness", Value: "10"},
+		},
+		StoredSysctls: []Sysctl{{Name: "vm.swappiness", Value: "10"}},
+	}
+
+	cases := []struct {
+		name   string
+		scope  nodeSysctlScope
+		stored bool
+		want   []Sysctl
+	}{
+		{
+			name:   "profile resolved set includes inherited sysctls",
+			scope:  profile,
+			stored: false,
+			want:   profile.Sysctls,
+		},
+		{
+			name:   "profile stored map excludes inherited sysctls",
+			scope:  profile,
+			stored: true,
+			want:   profile.StoredSysctls,
+		},
+		{
+			name:   "global scope resolves to what it stores",
+			scope:  nodeSysctlScope{Sysctls: global, StoredSysctls: global},
+			stored: true,
+			want:   global,
+		},
+		{
+			name:   "profile storing nothing reports an empty map",
+			scope:  nodeSysctlScope{ProfileName: "edge-workers", Sysctls: global, StoredSysctls: []Sysctl{}},
+			stored: true,
+			want:   []Sysctl{},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := tc.scope.reportSysctls(tc.stored)
+			if len(got) != len(tc.want) {
+				t.Fatalf("reportSysctls(%t) = %v, want %v", tc.stored, got, tc.want)
+			}
+			for i := range got {
+				if got[i] != tc.want[i] {
+					t.Errorf("reportSysctls(%t)[%d] = %v, want %v", tc.stored, i, got[i], tc.want[i])
+				}
+			}
+		})
+	}
+}
+
+func TestFilterNodeSysctlScopes(t *testing.T) {
+	scopes := []nodeSysctlScope{
+		{ReleaseName: "dokku-node-sysctls-global"},
+		{ProfileName: "edge-workers", ReleaseName: "dokku-node-sysctls-profile-edge-workers"},
+		{ProfileName: "gpu-workers", ReleaseName: "dokku-node-sysctls-profile-gpu-workers"},
+	}
+
+	cases := []struct {
+		name        string
+		global      bool
+		profileName string
+		want        []string
+	}{
+		{
+			name: "no filter keeps every scope",
+			want: []string{"", "edge-workers", "gpu-workers"},
+		},
+		{
+			name:   "global keeps only the unprofiled scope",
+			global: true,
+			want:   []string{""},
+		},
+		{
+			name:        "profile keeps only its own scope",
+			profileName: "edge-workers",
+			want:        []string{"edge-workers"},
+		},
+		{
+			name:        "unknown profile keeps nothing",
+			profileName: "missing-profile",
+			want:        []string{},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := filterNodeSysctlScopes(scopes, tc.global, tc.profileName)
+			if len(got) != len(tc.want) {
+				t.Fatalf("filterNodeSysctlScopes() returned %d scopes, want %d", len(got), len(tc.want))
+			}
+			for i := range got {
+				if got[i].ProfileName != tc.want[i] {
+					t.Errorf("filterNodeSysctlScopes()[%d].ProfileName = %q, want %q", i, got[i].ProfileName, tc.want[i])
+				}
+			}
+		})
+	}
+}

plugins/scheduler-k3s/src/commands/commands.go

@@ -38,7 +38,7 @@ Additional commands:`
     scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE], Clear all node-level kernel sysctls for unprofiled nodes or a single node profile
     scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE], Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
     scheduler-k3s:node-sysctls:set --replace <sysctl=value> [<sysctl=value> ...] [--global|--profile PROFILE], Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile
-    scheduler-k3s:node-sysctls:report [--format stdout|json], Displays the node-level kernel sysctls applied to each scope
+    scheduler-k3s:node-sysctls:report [--format stdout|json] [--stored] [--global|--profile PROFILE], Displays the node-level kernel sysctls for each scope
     scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
     scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster
     scheduler-k3s:profiles:list [--format json|stdout], Lists all node profiles in the k3s cluster

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -257,8 +257,21 @@ func main() {
 	case "node-sysctls:report":
 		args := flag.NewFlagSet("scheduler-k3s:node-sysctls:report", flag.ExitOnError)
 		format := args.String("format", "stdout", "format: [ stdout | json ]")
+		global := args.Bool("global", false, "--global: report only on all nodes without a node profile")
+		profileName := args.String("profile", "", "--profile: report only on a node profile instead of every scope")
+		stored := args.Bool("stored", false, "--stored: report the sysctls each scope stores itself rather than the resolved set it applies")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandNodeSysctlsReport(*format)
+		if *global && *profileName != "" {
+			err = fmt.Errorf("Only one of --global and --profile may be specified")
+			break
+		}
+
+		err = scheduler_k3s.CommandNodeSysctlsReport(scheduler_k3s.NodeSysctlsReportInput{
+			Format:      *format,
+			Global:      *global,
+			ProfileName: *profileName,
+			Stored:      *stored,
+		})
 	case "preview":
 		args := flag.NewFlagSet("scheduler-k3s:preview", flag.ExitOnError)
 		context := args.Int("context", 3, "--context: number of unchanged lines of context around each change (-1 for full output)")

plugins/scheduler-k3s/subcommands.go

@@ -152,10 +152,29 @@ func warnUnmanagedSysctls(names []string) {
 	}
 }
 
-// CommandNodeSysctlsReport displays the configured node-level kernel sysctls
+// NodeSysctlsReportInput captures the inputs accepted by node-sysctls:report.
-func CommandNodeSysctlsReport(format string) error {
+// Global and ProfileName are mutually exclusive, and naming neither reports every
-	if format != "stdout" && format != "json" {
+// scope rather than the global one.
-		return fmt.Errorf("Invalid format: %s", format)
+type NodeSysctlsReportInput struct {
+	Format      string
+	Global      bool
+	ProfileName string
+	Stored      bool
+}
+
+// CommandNodeSysctlsReport displays the configured node-level kernel sysctls. A named
+// profile only has to exist to be reported on, unlike one being written to: a profile
+// whose legacy name cannot back a helm release still resolves to a scope, and the map
+// it stores is worth reading back even though nothing applies it.
+func CommandNodeSysctlsReport(input NodeSysctlsReportInput) error {
+	if input.Format != "stdout" && input.Format != "json" {
+		return fmt.Errorf("Invalid format: %s", input.Format)
+	}
+
+	if input.ProfileName != "" {
+		if err := verifyNodeProfileExists(input.ProfileName); err != nil {
+			return err
+		}
 	}
 
 	scopes, err := resolveNodeSysctlScopes()
@@ -163,19 +182,16 @@ func CommandNodeSysctlsReport(format string) error {
 		return err
 	}
 
-	if format == "json" {
+	scopes = filterNodeSysctlScopes(scopes, input.Global, input.ProfileName)
+
+	if input.Format == "json" {
 		output := map[string]map[string]string{}
 		for _, scope := range scopes {
-			key := scope.ProfileName
-			if key == "" {
-				key = "--global"
-			}
-
 			entries := map[string]string{}
-			for _, sysctl := range scope.Sysctls {
+			for _, sysctl := range scope.reportSysctls(input.Stored) {
 				entries[sysctl.Name] = sysctl.Value
 			}
-			output[key] = entries
+			output[nodeSysctlsScopeKey(scope.ProfileName)] = entries
 		}
 
 		b, err := json.Marshal(output)
@@ -189,12 +205,8 @@ func CommandNodeSysctlsReport(format string) error {
 
 	lines := []string{"scope|sysctl|value"}
 	for _, scope := range scopes {
-		scopeName := scope.ProfileName
+		scopeName := nodeSysctlsScopeKey(scope.ProfileName)
-		if scopeName == "" {
+		for _, sysctl := range scope.reportSysctls(input.Stored) {
-			scopeName = "--global"
-		}
-
-		for _, sysctl := range scope.Sysctls {
 			lines = append(lines, fmt.Sprintf("%s|%s|%s", scopeName, sysctl.Name, sysctl.Value))
 		}
 	}

tests/unit/scheduler-k3s-node-sysctls-2.bats

@@ -0,0 +1,91 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  global_setup
+}
+
+teardown() {
+  dokku scheduler-k3s:node-sysctls:clear --global >/dev/null 2>/dev/null || true
+  dokku scheduler-k3s:node-sysctls:clear --profile edge-workers >/dev/null 2>/dev/null || true
+  dokku scheduler-k3s:profiles:remove edge-workers >/dev/null 2>/dev/null || true
+  global_teardown
+}
+
+@test "(scheduler-k3s:node-sysctls:report) --stored returns only the map each scope owns" {
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.overcommit_memory=1 vm.swappiness=20"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:profiles:add edge-workers --role worker"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --profile edge-workers vm.swappiness=60"
+  assert_success
+
+  # the resolved set a profile's daemonset applies carries the global sysctls underneath its own
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"edge-workers\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'"
+  assert_success
+  assert_output "vm.overcommit_memory=1,vm.swappiness=60"
+
+  # the stored map holds only what the profile itself was set to
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --format json | jq -r '.\"edge-workers\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'"
+  assert_success
+  assert_output "vm.swappiness=60"
+
+  # the global scope stores exactly what it applies
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --format json | jq -r '.\"--global\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'"
+  assert_success
+  assert_output "vm.overcommit_memory=1,vm.swappiness=20"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --global --format json | jq -rS '.'"
+  assert_success
+  stored_global="$output"
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --global --format json | jq -rS '.'"
+  assert_success
+  assert_output "$stored_global"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers --format json | jq -rS 'keys | join(\",\")'"
+  assert_success
+  assert_output "edge-workers"
+
+  # the stdout renderer honours both flags too
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers"
+  assert_success
+  assert_output_contains "edge-workers"
+  assert_output_not_contains "vm.overcommit_memory"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --profile edge-workers"
+  assert_success
+
+  # a cleared profile keeps inheriting the global sysctls, so only the stored map converges
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"edge-workers\" | length'"
+  assert_success
+  assert_output "2"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers --format json | jq -r '.\"edge-workers\" | length'"
+  assert_success
+  assert_output "0"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --profile edge-workers"
+  assert_success
+  assert_output_not_contains "vm.overcommit_memory"
+  assert_output_not_contains "vm.swappiness"
+}
+
+@test "(scheduler-k3s:node-sysctls:report) rejects conflicting and unknown scopes" {
+  run /bin/bash -c "dokku scheduler-k3s:profiles:add edge-workers --role worker"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --global --profile edge-workers"
+  assert_failure
+  assert_output_contains "Only one of --global and --profile may be specified"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --profile missing-profile"
+  assert_failure
+  assert_output_contains "Node profile missing-profile not found"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --stored --format bogus"
+  assert_failure
+  assert_output_contains "Invalid format: bogus"
+}