Merge pull request #9051 from dokku/8990-add-a-whole-set-form-for-scheduler-k3s-annotations-and-labels

feat: add a whole-set form for scheduler-k3s metadata

docs/deployment/schedulers/k3s.md

@@ -4,9 +4,12 @@
 > New as of 0.33.0
 
 ```
+scheduler-k3s:annotations:clear <app|--global> [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Clear all annotations for an app or a single process-type/resource-type scope
 scheduler-k3s:annotations:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Set or clear an annotation for a given app/process-type/resource-type combination
+scheduler-k3s:annotations:set --replace <app|--global> <key=value> [<key=value> ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Replace the entire annotation map for a given app/process-type/resource-type combination
 scheduler-k3s:annotations:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Displays a scheduler-k3s annotations report for one or more apps
 scheduler-k3s:autoscaling-auth:set <app|--global> <trigger> [<--metadata key=value>...] # Set or clear a scheduler-k3s autoscaling keda trigger authentication resource for an app
+scheduler-k3s:autoscaling-auth:set --replace <app|--global> <trigger> <--metadata key=value> [<--metadata key=value>...] # Replace the entire metadata map for a scheduler-k3s autoscaling keda trigger authentication resource
 scheduler-k3s:autoscaling-auth:report [<app>|--global] [--format stdout|json] [--include-metadata] # Displays a scheduler-k3s autoscaling auth report for one or more apps
 scheduler-k3s:charts:report [<chart>] [--format stdout|json] # Displays a scheduler-k3s chart override report
 scheduler-k3s:charts:set <chart-name.property> (<value>) # Set or clear a chart-specific helm value
@@ -15,9 +18,13 @@ scheduler-k3s:cluster:list [--format json|stdout] # Lists all nodes in a Dokku-m
 scheduler-k3s:cluster:remove [node-id] # Removes client node to a Dokku-managed cluster
 scheduler-k3s:ensure-charts # Ensures the k3s charts are installed
 scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] # Initializes a cluster
+scheduler-k3s:labels:clear <app|--global> [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Clear all labels for an app or a single process-type/resource-type scope
 scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Set or clear a label for a given app/process-type/resource-type combination
+scheduler-k3s:labels:set --replace <app|--global> <key=value> [<key=value> ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Replace the entire label map for a given app/process-type/resource-type combination
 scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Displays a scheduler-k3s labels report for one or more apps
+scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE] # Clear all node-level kernel sysctls for unprofiled nodes or a single node profile
 scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE] # Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
+scheduler-k3s:node-sysctls:set --replace <sysctl=value> [<sysctl=value> ...] [--global|--profile PROFILE] # Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile
 scheduler-k3s:node-sysctls:report [--format stdout|json] # Displays the node-level kernel sysctls applied to each scope
 scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded] # Displays a diff between the current and next deployment for an app
 scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] # Adds a node profile to the k3s cluster
@@ -595,6 +602,49 @@ dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type d
 
 A `ps:restart` is required after removing annotations in order to remove them from running resources.
 
+#### Replacing the entire annotation set
+
+The per-key form above changes one annotation at a time, so matching the stored annotations to a declared set means computing the difference and issuing one command per added, changed and removed key. The `--replace` flag writes the whole map for a scope in a single call, taking `key=value` pairs instead of a single key and value.
+
+```shell
+dokku scheduler-k3s:annotations:set --replace node-js-app --resource-type deployment annotation.key=annotation.value other.key=other.value
+```
+
+Anything previously stored for that scope and not named in the call is removed. `--resource-type` is still required, and `--process-type` scopes the replacement the same way it scopes a single key.
+
+```shell
+dokku scheduler-k3s:annotations:set --replace node-js-app --resource-type deployment --process-type web annotation.key=annotation.value
+```
+
+Every pair is parsed before anything is written, so a rejected pair leaves the stored annotations untouched. A pair without a `=`, a pair with an empty key, and a key named more than once are all rejected.
+
+A pair ending in `=` stores an empty value, which the per-key form cannot express - there, an empty value deletes the key instead.
+
+```shell
+dokku scheduler-k3s:annotations:set --replace node-js-app --resource-type deployment annotation.key=
+```
+
+> [!NOTE]
+> An empty pair list is rejected rather than treated as a request to remove everything, so a generated list that expands to nothing cannot silently drop an app's annotations. Use `scheduler-k3s:annotations:clear` for that.
+
+#### Clearing annotations
+
+The `scheduler-k3s:annotations:clear` command removes every annotation configured for an app.
+
+```shell
+dokku scheduler-k3s:annotations:clear node-js-app
+```
+
+The `--process-type` and `--resource-type` flags narrow what is cleared, and filter the same way they do for `scheduler-k3s:annotations:report`. Omitting both clears every scope.
+
+```shell
+dokku scheduler-k3s:annotations:clear node-js-app --resource-type deployment
+dokku scheduler-k3s:annotations:clear node-js-app --process-type web --resource-type deployment
+dokku scheduler-k3s:annotations:clear --global --resource-type deployment
+```
+
+A `ps:restart` is required after clearing annotations in order to remove them from running resources.
+
 #### Displaying annotations
 
 Configured annotations can be inspected with the `scheduler-k3s:annotations:report` command. Without arguments, it iterates every app and prints all annotations. Passing an app name (or `--global`) scopes the report:
@@ -658,15 +708,38 @@ A `ps:restart` is required after setting labels in order to have them apply to r
 
 #### Removing a label
 
-To unset an label, pass an empty value:
+To unset a label, pass an empty value:
 
 ```shell
-dokku scheduler-k3s:annotations:set node-js-app label.key --resource-type deployment
+dokku scheduler-k3s:labels:set node-js-app label.key --resource-type deployment
 dokku scheduler-k3s:labels:set node-js-app label.key --resource-type deployment --process-type web
 ```
 
 A `ps:restart` is required after removing labels in order to remove them from running resources.
 
+#### Replacing the entire label set
+
+As with annotations, the `--replace` flag writes the whole map for a scope in a single call, taking `key=value` pairs instead of a single key and value.
+
+```shell
+dokku scheduler-k3s:labels:set --replace node-js-app --resource-type deployment label.key=label.value other.key=other.value
+```
+
+Anything previously stored for that scope and not named in the call is removed. `--resource-type` is still required, `--process-type` scopes the replacement, every pair is parsed before anything is written, and a pair ending in `=` stores an empty value. An empty pair list is rejected; use `scheduler-k3s:labels:clear` to remove everything.
+
+#### Clearing labels
+
+The `scheduler-k3s:labels:clear` command removes every label configured for an app, with the same optional `--process-type` and `--resource-type` filters as `scheduler-k3s:labels:report`.
+
+```shell
+dokku scheduler-k3s:labels:clear node-js-app
+dokku scheduler-k3s:labels:clear node-js-app --resource-type deployment
+dokku scheduler-k3s:labels:clear node-js-app --process-type web --resource-type deployment
+dokku scheduler-k3s:labels:clear --global --resource-type deployment
+```
+
+A `ps:restart` is required after clearing labels in order to remove them from running resources.
+
 #### Displaying labels
 
 Configured labels can be inspected with the `scheduler-k3s:labels:report` command. The surface mirrors `scheduler-k3s:annotations:report`:
@@ -809,9 +882,19 @@ After execution, Dokku will include the following resources for each specified t
 
 If the `--global` flag is specified instead of an app name, a custom helm chart is created on the fly with the above resources.
 
+##### Replacing Authentication Metadata
+
+Without `--replace`, the metadata supplied is merged into whatever is already stored for the trigger, so a key that is no longer named stays behind. The `--replace` flag drops the trigger's existing metadata before writing, making the stored set exactly what was specified.
+
+```shell
+dokku scheduler-k3s:autoscaling-auth:set --replace node-js-app datadog --metadata apiKey=1234567890 --metadata appKey=asdfghjkl
+```
+
+In the example above, a `datadogSite` key configured earlier is removed rather than left in place. At least one `--metadata` flag is required with `--replace`; omit the flag entirely to remove everything, as below.
+
 ##### Removing Authentication Resources
 
-To remove a configured authenticatin resource, run the `scheduler-k3s:autoscaling-auth:set` command with no metadata specified. Subsequent deploys will not include these resources.
+To remove a configured authentication resource, run the `scheduler-k3s:autoscaling-auth:set` command with no metadata specified. Subsequent deploys will not include these resources.
 
 ```shell
 dokku scheduler-k3s:autoscaling-auth:set $APP $TRIGGER_TYPE
@@ -908,6 +991,23 @@ A profile scope inherits everything set globally and overrides it on conflict, s
 
 A profile created by an older Dokku may carry a name outside the [current rules](#adding-profiles), such as one containing uppercase or longer than 26 characters. Dokku cannot name a helm release after such a profile, so it is skipped with a warning and `node-sysctls:set --profile` refuses it. Recreate the profile under a valid name to give it sysctls; `scheduler-k3s:profiles:remove` still removes the old one.
 
+The `--replace` flag writes the whole set for a scope in one call, taking `sysctl=value` pairs instead of a single name and value. Anything previously set for that scope and not named in the call is cleared, with the same caveat about the value staying on affected nodes until they reboot.
+
+```shell
+dokku scheduler-k3s:node-sysctls:set --replace --global vm.max_map_count=262144 vm.swappiness=10
+```
+
+Every pair is parsed before anything is written, so a rejected pair leaves the stored sysctls untouched. A pair without a `=`, a pair with an empty key, and a key named more than once are all rejected. An empty pair list is rejected rather than treated as a request to remove everything.
+
+To remove every sysctl for a scope, use `scheduler-k3s:node-sysctls:clear`.
+
+```shell
+dokku scheduler-k3s:node-sysctls:clear --global
+dokku scheduler-k3s:node-sysctls:clear --profile edge-workers
+```
+
+Clearing a scope removes its DaemonSet. As with clearing a single sysctl, the values it last wrote stay in place on affected nodes until they reboot.
+
 Use `node-sysctls:report` to see the resolved set for every scope.
 
 ```shell

plugins/scheduler-k3s/Makefile

@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
+SUBCOMMANDS = subcommands/annotations:clear subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:clear subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:clear subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
 TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/scheduler-uses-host-cron triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s

plugins/scheduler-k3s/metadata.go

@@ -0,0 +1,275 @@
+package scheduler_k3s
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// metadataField selects which of the scheduler-k3s metadata maps a command
+// operates on, so the annotations and labels commands can share one
+// implementation the way the storage plugin does.
+type metadataField struct {
+	// Name is the plural noun used in user-facing messages.
+	Name string
+	// Singular is the singular noun used in user-facing messages.
+	Singular string
+	// ClearCommand is the command an empty --replace list points the user at.
+	ClearCommand string
+	// PropertyPrefix namespaces the property names this field owns. Annotations
+	// are stored unprefixed, which is why annotations alone must skip the
+	// property names reserved by other scheduler-k3s features.
+	PropertyPrefix string
+	// SkipReserved drops property names owned by other scheduler-k3s features.
+	SkipReserved bool
+	// ResourceTypes lists the kubernetes resource types this field supports.
+	ResourceTypes []string
+}
+
+var annotationsField = metadataField{
+	Name:          "annotations",
+	Singular:      "annotation",
+	ClearCommand:  "scheduler-k3s:annotations:clear",
+	SkipReserved:  true,
+	ResourceTypes: AnnotationResourceTypes,
+}
+
+var labelsField = metadataField{
+	Name:           "labels",
+	Singular:       "label",
+	ClearCommand:   "scheduler-k3s:labels:clear",
+	PropertyPrefix: "labels.",
+	ResourceTypes:  LabelResourceTypes,
+}
+
+// MetadataSetInput captures the inputs accepted by annotations:set and labels:set.
+// Key/Value and Pairs are mutually exclusive: Pairs is populated only under
+// --replace, where the positional arguments are key=value pairs rather than a
+// single key and value.
+type MetadataSetInput struct {
+	AppName      string
+	ProcessType  string
+	ResourceType string
+	Key          string
+	Value        string
+	Pairs        []string
+	Replace      bool
+}
+
+// MetadataClearInput captures the inputs accepted by annotations:clear and labels:clear.
+// An empty ProcessType or ResourceType filters nothing, matching the report commands.
+type MetadataClearInput struct {
+	AppName      string
+	ProcessType  string
+	ResourceType string
+}
+
+// metadataPropertyName returns the property a field stores one scope's map under.
+func metadataPropertyName(field metadataField, processType string, resourceType string) string {
+	return fmt.Sprintf("%s%s.%s", field.PropertyPrefix, processType, resourceType)
+}
+
+// verifyMetadataAppName applies the standard app name check, exempting the global scope.
+func verifyMetadataAppName(appName string) error {
+	if appName == "--global" {
+		return nil
+	}
+
+	return common.VerifyAppName(appName)
+}
+
+// validateResourceType reports whether a resource type is one the field supports.
+// An empty resource type passes, as the clear commands treat it as an absent filter;
+// callers that require one check for it separately.
+func validateResourceType(field metadataField, resourceType string) error {
+	if resourceType == "" {
+		return nil
+	}
+
+	for _, valid := range field.ResourceTypes {
+		if valid == resourceType {
+			return nil
+		}
+	}
+
+	valid := make([]string, len(field.ResourceTypes))
+	copy(valid, field.ResourceTypes)
+	sort.Strings(valid)
+	return fmt.Errorf("Invalid resource-type specified, valid resource types include: %s", strings.Join(valid, ", "))
+}
+
+// parseMetadataPairs turns key=value arguments into a map. A pair without a '='
+// or with an empty key is rejected, as is a key specified more than once, since
+// a repeated key leaves the declared map ambiguous. A pair ending in '=' stores
+// an empty value, which the per-key form cannot express.
+func parseMetadataPairs(pairs []string) (map[string]string, error) {
+	parsed := map[string]string{}
+	for _, pair := range pairs {
+		parts := strings.SplitN(pair, "=", 2)
+		if len(parts) != 2 || parts[0] == "" {
+			return nil, fmt.Errorf("Invalid key=value pair: %s", pair)
+		}
+
+		if _, ok := parsed[parts[0]]; ok {
+			return nil, fmt.Errorf("Duplicate key specified: %s", parts[0])
+		}
+
+		parsed[parts[0]] = parts[1]
+	}
+
+	return parsed, nil
+}
+
+// setMetadataKey writes a single key on one of the metadata maps. An empty value
+// deletes just that key, leaving its siblings in place.
+func setMetadataKey(field metadataField, input MetadataSetInput) error {
+	property := metadataPropertyName(field, input.ProcessType, input.ResourceType)
+	if input.Value == "" {
+		if err := common.PropertyMapDelete("scheduler-k3s", input.AppName, property, input.Key); err != nil {
+			return fmt.Errorf("Unable to delete property map entry: %w", err)
+		}
+		return nil
+	}
+
+	if err := common.PropertyMapSet("scheduler-k3s", input.AppName, property, input.Key, input.Value); err != nil {
+		return fmt.Errorf("Unable to set property map entry: %w", err)
+	}
+
+	return nil
+}
+
+// replaceMetadata swaps one scope's whole map for the declared one in a single write.
+// Every pair is parsed before anything is written, so a rejected pair leaves the
+// stored map untouched.
+func replaceMetadata(field metadataField, input MetadataSetInput) error {
+	if len(input.Pairs) == 0 {
+		return fmt.Errorf("Must specify at least one key=value pair, use %s to remove all %s", field.ClearCommand, field.Name)
+	}
+
+	parsed, err := parseMetadataPairs(input.Pairs)
+	if err != nil {
+		return err
+	}
+
+	property := metadataPropertyName(field, input.ProcessType, input.ResourceType)
+	if err := common.PropertyMapWrite("scheduler-k3s", input.AppName, property, parsed); err != nil {
+		return fmt.Errorf("Unable to write property map: %w", err)
+	}
+
+	return nil
+}
+
+// clearMetadata removes every metadata property matching the given filters.
+func clearMetadata(field metadataField, input MetadataClearInput) error {
+	properties, err := matchingMetadataProperties(field, input.AppName, input.ProcessType, input.ResourceType)
+	if err != nil {
+		return err
+	}
+
+	for _, property := range properties {
+		if err := common.PropertyDelete("scheduler-k3s", input.AppName, property.PropertyName); err != nil {
+			return fmt.Errorf("Unable to delete property: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// metadataProperty is one stored metadata scope, as found by scanning the property store.
+type metadataProperty struct {
+	// PropertyName is the name the scope is stored under, prefix included.
+	PropertyName string
+	ProcessType  string
+	ResourceType string
+}
+
+// matchingMetadataProperties scans the property store for the scopes a field owns on
+// appName, optionally filtered by processType/resourceType. Both the report commands
+// and the clear commands read through this, so they cannot disagree about what is in scope.
+func matchingMetadataProperties(field metadataField, appName string, processType string, resourceType string) ([]metadataProperty, error) {
+	properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, field.PropertyPrefix)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to get property list: %w", err)
+	}
+
+	knownResourceTypes := map[string]bool{}
+	for _, rt := range field.ResourceTypes {
+		knownResourceTypes[rt] = true
+	}
+
+	matching := []metadataProperty{}
+	for propertyName := range properties {
+		if field.SkipReserved && isReservedAnnotationProperty(propertyName) {
+			continue
+		}
+
+		suffix := strings.TrimPrefix(propertyName, field.PropertyPrefix)
+		dot := strings.LastIndex(suffix, ".")
+		if dot <= 0 || dot == len(suffix)-1 {
+			continue
+		}
+
+		propProcessType := suffix[:dot]
+		propResourceType := suffix[dot+1:]
+		if !knownResourceTypes[propResourceType] {
+			continue
+		}
+
+		if processType != "" && propProcessType != processType {
+			continue
+		}
+		if resourceType != "" && propResourceType != resourceType {
+			continue
+		}
+
+		matching = append(matching, metadataProperty{
+			PropertyName: propertyName,
+			ProcessType:  propProcessType,
+			ResourceType: propResourceType,
+		})
+	}
+
+	return matching, nil
+}
+
+// commandMetadataSet implements the annotations:set and labels:set commands.
+func commandMetadataSet(field metadataField, input MetadataSetInput) error {
+	if err := verifyMetadataAppName(input.AppName); err != nil {
+		return err
+	}
+
+	if input.ResourceType == "" {
+		return fmt.Errorf("Missing resource-type")
+	}
+
+	if err := validateResourceType(field, input.ResourceType); err != nil {
+		return err
+	}
+
+	if input.ProcessType == "" {
+		input.ProcessType = GlobalProcessType
+	}
+
+	if input.Replace {
+		return replaceMetadata(field, input)
+	}
+
+	return setMetadataKey(field, input)
+}
+
+// commandMetadataClear implements the annotations:clear and labels:clear commands.
+// Unlike the set commands, an omitted process-type or resource-type is an absent
+// filter rather than a default scope, matching the report commands.
+func commandMetadataClear(field metadataField, input MetadataClearInput) error {
+	if err := verifyMetadataAppName(input.AppName); err != nil {
+		return err
+	}
+
+	if err := validateResourceType(field, input.ResourceType); err != nil {
+		return err
+	}
+
+	return clearMetadata(field, input)
+}

plugins/scheduler-k3s/metadata_test.go

@@ -0,0 +1,188 @@
+package scheduler_k3s
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParseMetadataPairs(t *testing.T) {
+	tests := []struct {
+		name  string
+		pairs []string
+		want  map[string]string
+	}{
+		{
+			name:  "empty list",
+			pairs: []string{},
+			want:  map[string]string{},
+		},
+		{
+			name:  "multiple pairs",
+			pairs: []string{"foo=bar", "baz=qux"},
+			want:  map[string]string{"foo": "bar", "baz": "qux"},
+		},
+		{
+			name:  "trailing equals stores an empty value",
+			pairs: []string{"foo="},
+			want:  map[string]string{"foo": ""},
+		},
+		{
+			name:  "value containing equals is kept whole",
+			pairs: []string{"foo=a=b"},
+			want:  map[string]string{"foo": "a=b"},
+		},
+		{
+			name:  "kubernetes style key",
+			pairs: []string{"prometheus.io/scrape=true"},
+			want:  map[string]string{"prometheus.io/scrape": "true"},
+		},
+		{
+			name:  "multi-line value",
+			pairs: []string{"foo=line one\nline two"},
+			want:  map[string]string{"foo": "line one\nline two"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseMetadataPairs(test.pairs)
+			if err != nil {
+				t.Fatalf("parseMetadataPairs(%q) returned error: %s", test.pairs, err)
+			}
+			if !reflect.DeepEqual(got, test.want) {
+				t.Errorf("parseMetadataPairs(%q) = %v, want %v", test.pairs, got, test.want)
+			}
+		})
+	}
+}
+
+func TestParseMetadataPairsErrors(t *testing.T) {
+	tests := []struct {
+		name  string
+		pairs []string
+		want  string
+	}{
+		{
+			name:  "missing equals",
+			pairs: []string{"foo"},
+			want:  "Invalid key=value pair: foo",
+		},
+		{
+			name:  "empty key",
+			pairs: []string{"=bar"},
+			want:  "Invalid key=value pair: =bar",
+		},
+		{
+			name:  "empty pair",
+			pairs: []string{""},
+			want:  "Invalid key=value pair: ",
+		},
+		{
+			name:  "duplicate key",
+			pairs: []string{"foo=bar", "foo=qux"},
+			want:  "Duplicate key specified: foo",
+		},
+		{
+			name:  "duplicate key with identical value",
+			pairs: []string{"foo=bar", "foo=bar"},
+			want:  "Duplicate key specified: foo",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseMetadataPairs(test.pairs)
+			if err == nil {
+				t.Fatalf("parseMetadataPairs(%q) = %v, want error", test.pairs, got)
+			}
+			if err.Error() != test.want {
+				t.Errorf("parseMetadataPairs(%q) error = %q, want %q", test.pairs, err.Error(), test.want)
+			}
+		})
+	}
+}
+
+func TestValidateResourceType(t *testing.T) {
+	tests := []struct {
+		name         string
+		field        metadataField
+		resourceType string
+		wantErr      bool
+	}{
+		{
+			name:         "valid annotation resource type",
+			field:        annotationsField,
+			resourceType: "deployment",
+		},
+		{
+			name:         "valid label resource type",
+			field:        labelsField,
+			resourceType: "deployment",
+		},
+		{
+			// An empty resource type is an absent filter for the clear commands.
+			// The set commands reject it separately.
+			name:         "empty resource type passes",
+			field:        annotationsField,
+			resourceType: "",
+		},
+		{
+			name:         "unknown resource type",
+			field:        annotationsField,
+			resourceType: "deploymnet",
+			wantErr:      true,
+		},
+		{
+			// Keda resources do not flow user labels through, so they are
+			// annotation-only resource types.
+			name:         "keda resource type is valid for annotations",
+			field:        annotationsField,
+			resourceType: "keda_scaled_object",
+		},
+		{
+			name:         "keda resource type is invalid for labels",
+			field:        labelsField,
+			resourceType: "keda_scaled_object",
+			wantErr:      true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := validateResourceType(test.field, test.resourceType)
+			if test.wantErr {
+				if err == nil {
+					t.Fatalf("validateResourceType(%s, %q) = nil, want error", test.field.Name, test.resourceType)
+				}
+				if !strings.Contains(err.Error(), "Invalid resource-type specified, valid resource types include: ") {
+					t.Errorf("validateResourceType(%s, %q) error = %q, want the valid type list", test.field.Name, test.resourceType, err.Error())
+				}
+				for _, valid := range test.field.ResourceTypes {
+					if !strings.Contains(err.Error(), valid) {
+						t.Errorf("validateResourceType(%s, %q) error = %q, missing valid type %s", test.field.Name, test.resourceType, err.Error(), valid)
+					}
+				}
+				return
+			}
+
+			if err != nil {
+				t.Errorf("validateResourceType(%s, %q) returned error: %s", test.field.Name, test.resourceType, err)
+			}
+		})
+	}
+}
+
+func TestMetadataPropertyName(t *testing.T) {
+	// Annotations are stored unprefixed and labels under a labels. prefix, which is
+	// what getAnnotation and getLabel already build.
+	if got := metadataPropertyName(annotationsField, "web", "deployment"); got != "web.deployment" {
+		t.Errorf("metadataPropertyName(annotations, web, deployment) = %q, want web.deployment", got)
+	}
+	if got := metadataPropertyName(labelsField, "web", "deployment"); got != "labels.web.deployment" {
+		t.Errorf("metadataPropertyName(labels, web, deployment) = %q, want labels.web.deployment", got)
+	}
+	if got := metadataPropertyName(annotationsField, GlobalProcessType, "pod"); got != "--global.pod" {
+		t.Errorf("metadataPropertyName(annotations, --global, pod) = %q, want --global.pod", got)
+	}
+}

plugins/scheduler-k3s/report.go

@@ -396,101 +396,34 @@ func ReportLabelsSingleApp(appName string, format string, processType string, re
 // collectAnnotationsEntries scans the property store for annotation entries on appName,
 // optionally filtered by processType/resourceType.
 func collectAnnotationsEntries(appName string, processType string, resourceType string) ([]annotationsLabelsReportEntry, error) {
-	properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, "")
+	return collectMetadataEntries(annotationsField, appName, processType, resourceType, getAnnotation)
-	if err != nil {
-		return nil, fmt.Errorf("Unable to get property list: %w", err)
-	}
-
-	knownResourceTypes := map[string]bool{}
-	for _, rt := range AnnotationResourceTypes {
-		knownResourceTypes[rt] = true
-	}
-
-	entries := []annotationsLabelsReportEntry{}
-	for propertyName := range properties {
-		if isReservedAnnotationProperty(propertyName) {
-			continue
-		}
-
-		dot := strings.LastIndex(propertyName, ".")
-		if dot <= 0 || dot == len(propertyName)-1 {
-			continue
-		}
-
-		propProcessType := propertyName[:dot]
-		propResourceType := propertyName[dot+1:]
-		if !knownResourceTypes[propResourceType] {
-			continue
-		}
-
-		if processType != "" && propProcessType != processType {
-			continue
-		}
-		if resourceType != "" && propResourceType != resourceType {
-			continue
-		}
-
-		annotationMap, err := getAnnotation(appName, propProcessType, propResourceType)
-		if err != nil {
-			return nil, fmt.Errorf("Unable to read annotation %s: %w", propertyName, err)
-		}
-
-		for key, value := range annotationMap {
-			entries = append(entries, annotationsLabelsReportEntry{
-				ProcessType:  propProcessType,
-				ResourceType: propResourceType,
-				Key:          key,
-				Value:        value,
-			})
-		}
-	}
-
-	return entries, nil
 }
 
 // collectLabelsEntries scans the property store for label entries on appName,
 // optionally filtered by processType/resourceType.
 func collectLabelsEntries(appName string, processType string, resourceType string) ([]annotationsLabelsReportEntry, error) {
-	properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, "labels.")
+	return collectMetadataEntries(labelsField, appName, processType, resourceType, getLabel)
-	if err != nil {
+}
-		return nil, fmt.Errorf("Unable to get property list: %w", err)
-	}
 
-	knownResourceTypes := map[string]bool{}
+// collectMetadataEntries flattens every stored scope a field owns on appName into
-	for _, rt := range LabelResourceTypes {
+// report entries, reading the scopes through the same scan the clear commands use.
-		knownResourceTypes[rt] = true
+func collectMetadataEntries(field metadataField, appName string, processType string, resourceType string, read func(string, string, string) (map[string]string, error)) ([]annotationsLabelsReportEntry, error) {
+	properties, err := matchingMetadataProperties(field, appName, processType, resourceType)
+	if err != nil {
+		return nil, err
 	}
 
 	entries := []annotationsLabelsReportEntry{}
-	for propertyName := range properties {
+	for _, property := range properties {
-		suffix := strings.TrimPrefix(propertyName, "labels.")
+		values, err := read(appName, property.ProcessType, property.ResourceType)
-		dot := strings.LastIndex(suffix, ".")
-		if dot <= 0 || dot == len(suffix)-1 {
-			continue
-		}
-
-		propProcessType := suffix[:dot]
-		propResourceType := suffix[dot+1:]
-		if !knownResourceTypes[propResourceType] {
-			continue
-		}
-
-		if processType != "" && propProcessType != processType {
-			continue
-		}
-		if resourceType != "" && propResourceType != resourceType {
-			continue
-		}
-
-		labelMap, err := getLabel(appName, propProcessType, propResourceType)
 		if err != nil {
-			return nil, fmt.Errorf("Unable to read label %s: %w", propertyName, err)
+			return nil, fmt.Errorf("Unable to read %s %s: %w", field.Singular, property.PropertyName, err)
 		}
 
-		for key, value := range labelMap {
+		for key, value := range values {
 			entries = append(entries, annotationsLabelsReportEntry{
-				ProcessType:  propProcessType,
+				ProcessType:  property.ProcessType,
-				ResourceType: propResourceType,
+				ResourceType: property.ResourceType,
 				Key:          key,
 				Value:        value,
 			})

plugins/scheduler-k3s/src/commands/commands.go

@@ -17,9 +17,12 @@ Manage scheduler-k3s settings for an app
 Additional commands:`
 
 	helpContent = `
+    scheduler-k3s:annotations:clear <app|--global> [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Clear all annotations for an app or a single process-type/resource-type scope
     scheduler-k3s:annotations:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear an annotation for a given app/process-type/resource-type combination
+    scheduler-k3s:annotations:set --replace <app|--global> <key=value> [<key=value> ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Replace the entire annotation map for a given app/process-type/resource-type combination
     scheduler-k3s:annotations:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s annotations report for one or more apps
     scheduler-k3s:autoscaling-auth:set <app|--global> <trigger> [<--metadata key=value>...], Set or clear a scheduler-k3s autoscaling keda trigger authentication resource for an app
+    scheduler-k3s:autoscaling-auth:set --replace <app|--global> <trigger> <--metadata key=value> [<--metadata key=value>...], Replace the entire metadata map for a scheduler-k3s autoscaling keda trigger authentication resource
     scheduler-k3s:autoscaling-auth:report [<app>|--global] [--format stdout|json] [--include-metadata], Displays a scheduler-k3s autoscaling auth report for one or more apps
     scheduler-k3s:charts:report [<chart>] [--format stdout|json], Displays a scheduler-k3s chart override report
     scheduler-k3s:charts:set <chart-name.property> (<value>), Set or clear a chart-specific helm value
@@ -28,9 +31,13 @@ Additional commands:`
     scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster
     scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
     scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster
+    scheduler-k3s:labels:clear <app|--global> [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Clear all labels for an app or a single process-type/resource-type scope
     scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
+    scheduler-k3s:labels:set --replace <app|--global> <key=value> [<key=value> ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Replace the entire label map for a given app/process-type/resource-type combination
     scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
+    scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE], Clear all node-level kernel sysctls for unprofiled nodes or a single node profile
     scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE], Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
+    scheduler-k3s:node-sysctls:set --replace <sysctl=value> [<sysctl=value> ...] [--global|--profile PROFILE], Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile
     scheduler-k3s:node-sysctls:report [--format stdout|json], Displays the node-level kernel sysctls applied to each scope
     scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
     scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -18,22 +18,50 @@ func main() {
 
 	var err error
 	switch subcommand {
+	case "annotations:clear":
+		args := flag.NewFlagSet("scheduler-k3s:annotations:clear", flag.ExitOnError)
+		global := args.Bool("global", false, "--global: clear global annotations")
+		processType := args.String("process-type", "", "--process-type: filter by process-type")
+		resourceType := args.String("resource-type", "", "--resource-type: filter by resource-type")
+		args.Parse(os.Args[2:])
+		appName := args.Arg(0)
+		if *global {
+			appName = "--global"
+		}
+
+		err = scheduler_k3s.CommandAnnotationsClear(scheduler_k3s.MetadataClearInput{
+			AppName:      appName,
+			ProcessType:  *processType,
+			ResourceType: *resourceType,
+		})
 	case "annotations:set":
 		args := flag.NewFlagSet("scheduler-k3s:annotations:set", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: set a global property")
 		processType := args.String("process-type", "", "--process-type: scope to process-type")
 		resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type")
+		replace := args.Bool("replace", false, "--replace: replace the entire annotation map for the scope")
 		args.Parse(os.Args[2:])
-		appName := args.Arg(0)
+		input := scheduler_k3s.MetadataSetInput{
-		property := args.Arg(1)
+			AppName:      args.Arg(0),
-		value := args.Arg(2)
+			ProcessType:  *processType,
-		if *global {
+			ResourceType: *resourceType,
-			appName = "--global"
+			Replace:      *replace,
-			property = args.Arg(0)
-			value = args.Arg(1)
 		}
 
-		err = scheduler_k3s.CommandAnnotationsSet(appName, *processType, *resourceType, property, value)
+		skip := 1
+		if *global {
+			input.AppName = "--global"
+			skip = 0
+		}
+
+		if *replace {
+			input.Pairs = common.VarArgs(args.Args(), skip)
+		} else {
+			input.Key = args.Arg(skip)
+			input.Value = args.Arg(skip + 1)
+		}
+
+		err = scheduler_k3s.CommandAnnotationsSet(input)
 	case "annotations:report":
 		args := flag.NewFlagSet("scheduler-k3s:annotations:report", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: show the global report")
@@ -55,6 +83,7 @@ func main() {
 		args := flag.NewFlagSet("scheduler-k3s:autoscaling-auth:set", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: set a global property")
 		metadata := args.StringToString("metadata", map[string]string{}, "--metadata: a key=value map of parameter metadata")
+		replace := args.Bool("replace", false, "--replace: replace the entire metadata map for the trigger")
 		args.Parse(os.Args[2:])
 		appName := args.Arg(0)
 		trigger := args.Arg(1)
@@ -62,7 +91,7 @@ func main() {
 			appName = "--global"
 			trigger = args.Arg(0)
 		}
-		err = scheduler_k3s.CommandAutoscalingAuthSet(appName, trigger, *metadata, *global)
+		err = scheduler_k3s.CommandAutoscalingAuthSet(appName, trigger, *metadata, *global, *replace)
 	case "autoscaling-auth:report":
 		args := flag.NewFlagSet("scheduler-k3s:autoscaling-auth:report", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: show the global report")
@@ -131,22 +160,50 @@ func main() {
 		kubeletArgs := args.StringSlice("kubelet-args", []string{}, "kubelet-args: repeatable key=value kubelet arguments (e.g., --kubelet-args key=value)")
 		args.Parse(os.Args[2:])
 		err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling, *kubeletArgs)
+	case "labels:clear":
+		args := flag.NewFlagSet("scheduler-k3s:labels:clear", flag.ExitOnError)
+		global := args.Bool("global", false, "--global: clear global labels")
+		processType := args.String("process-type", "", "--process-type: filter by process-type")
+		resourceType := args.String("resource-type", "", "--resource-type: filter by resource-type")
+		args.Parse(os.Args[2:])
+		appName := args.Arg(0)
+		if *global {
+			appName = "--global"
+		}
+
+		err = scheduler_k3s.CommandLabelsClear(scheduler_k3s.MetadataClearInput{
+			AppName:      appName,
+			ProcessType:  *processType,
+			ResourceType: *resourceType,
+		})
 	case "labels:set":
 		args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: set a global property")
 		processType := args.String("process-type", "", "--process-type: scope to process-type")
 		resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type")
+		replace := args.Bool("replace", false, "--replace: replace the entire label map for the scope")
 		args.Parse(os.Args[2:])
-		appName := args.Arg(0)
+		input := scheduler_k3s.MetadataSetInput{
-		property := args.Arg(1)
+			AppName:      args.Arg(0),
-		value := args.Arg(2)
+			ProcessType:  *processType,
-		if *global {
+			ResourceType: *resourceType,
-			appName = "--global"
+			Replace:      *replace,
-			property = args.Arg(0)
-			value = args.Arg(1)
 		}
 
-		err = scheduler_k3s.CommandLabelsSet(appName, *processType, *resourceType, property, value)
+		skip := 1
+		if *global {
+			input.AppName = "--global"
+			skip = 0
+		}
+
+		if *replace {
+			input.Pairs = common.VarArgs(args.Args(), skip)
+		} else {
+			input.Key = args.Arg(skip)
+			input.Value = args.Arg(skip + 1)
+		}
+
+		err = scheduler_k3s.CommandLabelsSet(input)
 	case "labels:report":
 		args := flag.NewFlagSet("scheduler-k3s:labels:report", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: show the global report")
@@ -164,18 +221,39 @@ func main() {
 			appName = "--global"
 		}
 		err = scheduler_k3s.CommandLabelsReport(appName, *format, *processType, *resourceType, infoFlag)
-	case "node-sysctls:set":
+	case "node-sysctls:clear":
-		args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError)
+		args := flag.NewFlagSet("scheduler-k3s:node-sysctls:clear", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: scope to all nodes without a node profile")
 		profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes")
 		args.Parse(os.Args[2:])
-		key := args.Arg(0)
-		value := args.Arg(1)
 		if *global && *profileName != "" {
 			err = fmt.Errorf("Only one of --global and --profile may be specified")
 			break
 		}
-		err = scheduler_k3s.CommandNodeSysctlsSet(*profileName, key, value)
+		err = scheduler_k3s.CommandNodeSysctlsClear(*profileName)
+	case "node-sysctls:set":
+		args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError)
+		global := args.Bool("global", false, "--global: scope to all nodes without a node profile")
+		profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes")
+		replace := args.Bool("replace", false, "--replace: replace the entire sysctl map for the scope")
+		args.Parse(os.Args[2:])
+		if *global && *profileName != "" {
+			err = fmt.Errorf("Only one of --global and --profile may be specified")
+			break
+		}
+
+		input := scheduler_k3s.NodeSysctlsSetInput{
+			ProfileName: *profileName,
+			Replace:     *replace,
+		}
+		if *replace {
+			input.Pairs = args.Args()
+		} else {
+			input.Key = args.Arg(0)
+			input.Value = args.Arg(1)
+		}
+
+		err = scheduler_k3s.CommandNodeSysctlsSet(input)
 	case "node-sysctls:report":
 		args := flag.NewFlagSet("scheduler-k3s:node-sysctls:report", flag.ExitOnError)
 		format := args.String("format", "stdout", "format: [ stdout | json ]")

plugins/scheduler-k3s/subcommands.go

@@ -20,56 +20,70 @@ import (
 	"helm.sh/helm/v3/pkg/chartutil"
 )
 
-// CommandAnnotationsSet set or clear a scheduler-k3s annotation for an app
+// CommandAnnotationsSet sets, clears or replaces scheduler-k3s annotations for an app
-func CommandAnnotationsSet(appName string, processType string, resourceType string, key string, value string) error {
+func CommandAnnotationsSet(input MetadataSetInput) error {
-	if resourceType == "" {
+	return commandMetadataSet(annotationsField, input)
-		return fmt.Errorf("Missing resource-type")
-	}
-
-	if processType == "" {
-		processType = GlobalProcessType
-	}
-
-	property := fmt.Sprintf("%s.%s", processType, resourceType)
-	if value == "" {
-		if err := common.PropertyMapDelete("scheduler-k3s", appName, property, key); err != nil {
-			return fmt.Errorf("Unable to delete property map entry: %w", err)
-		}
-		return nil
-	}
-
-	if err := common.PropertyMapSet("scheduler-k3s", appName, property, key, value); err != nil {
-		return fmt.Errorf("Unable to set property map entry: %w", err)
-	}
-
-	return nil
 }
 
-// CommandNodeSysctlsSet sets or clears a node-level kernel sysctl for a scope
+// CommandAnnotationsClear clears scheduler-k3s annotations for an app
-func CommandNodeSysctlsSet(profileName string, key string, value string) error {
+func CommandAnnotationsClear(input MetadataClearInput) error {
-	if key == "" {
+	return commandMetadataClear(annotationsField, input)
+}
+
+// NodeSysctlsSetInput captures the inputs accepted by node-sysctls:set.
+// Key/Value and Pairs are mutually exclusive: Pairs is populated only under
+// --replace, where the positional arguments are key=value pairs rather than a
+// single sysctl name and value.
+type NodeSysctlsSetInput struct {
+	ProfileName string
+	Key         string
+	Value       string
+	Pairs       []string
+	Replace     bool
+}
+
+// CommandNodeSysctlsSet sets, clears or replaces node-level kernel sysctls for a scope
+func CommandNodeSysctlsSet(input NodeSysctlsSetInput) error {
+	if err := verifyNodeSysctlsScope(input.ProfileName); err != nil {
+		return err
+	}
+
+	property := getNodeSysctlsProperty(input.ProfileName)
+	if input.Replace {
+		if len(input.Pairs) == 0 {
+			return fmt.Errorf("Must specify at least one key=value pair, use scheduler-k3s:node-sysctls:clear to remove all sysctls")
+		}
+
+		parsed, err := parseMetadataPairs(input.Pairs)
+		if err != nil {
+			return err
+		}
+
+		removed, err := removedSysctlNames(property, parsed)
+		if err != nil {
+			return err
+		}
+
+		if err := common.PropertyMapWrite("scheduler-k3s", "--global", property, parsed); err != nil {
+			return fmt.Errorf("Unable to write property map: %w", err)
+		}
+
+		warnUnmanagedSysctls(removed)
+		return CreateOrUpdateNodeSysctls(context.Background())
+	}
+
+	if input.Key == "" {
 		return fmt.Errorf("Missing sysctl name")
 	}
 
-	if profileName != "" {
+	if input.Value == "" {
-		if err := verifyNodeProfileExists(profileName); err != nil {
+		if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, input.Key); err != nil {
-			return err
-		}
-
-		if err := verifyNodeProfileSysctlsSupported(profileName); err != nil {
-			return err
-		}
-	}
-
-	property := getNodeSysctlsProperty(profileName)
-	if value == "" {
-		if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, key); err != nil {
 			return fmt.Errorf("Unable to delete property map entry: %w", err)
 		}
 
-		common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", key))
+		warnUnmanagedSysctls([]string{input.Key})
 	} else {
-		if err := common.PropertyMapSet("scheduler-k3s", "--global", property, key, value); err != nil {
+		if err := common.PropertyMapSet("scheduler-k3s", "--global", property, input.Key, input.Value); err != nil {
 			return fmt.Errorf("Unable to set property map entry: %w", err)
 		}
 	}
@@ -77,6 +91,68 @@ func CommandNodeSysctlsSet(profileName string, key string, value string) error {
 	return CreateOrUpdateNodeSysctls(context.Background())
 }
 
+// CommandNodeSysctlsClear clears every node-level kernel sysctl for a scope
+func CommandNodeSysctlsClear(profileName string) error {
+	if err := verifyNodeSysctlsScope(profileName); err != nil {
+		return err
+	}
+
+	property := getNodeSysctlsProperty(profileName)
+	removed, err := removedSysctlNames(property, map[string]string{})
+	if err != nil {
+		return err
+	}
+
+	if err := common.PropertyDelete("scheduler-k3s", "--global", property); err != nil {
+		return fmt.Errorf("Unable to delete property: %w", err)
+	}
+
+	warnUnmanagedSysctls(removed)
+	return CreateOrUpdateNodeSysctls(context.Background())
+}
+
+// verifyNodeSysctlsScope returns an error when a profile-scoped command names a profile
+// that does not exist or cannot back a helm release.
+func verifyNodeSysctlsScope(profileName string) error {
+	if profileName == "" {
+		return nil
+	}
+
+	if err := verifyNodeProfileExists(profileName); err != nil {
+		return err
+	}
+
+	return verifyNodeProfileSysctlsSupported(profileName)
+}
+
+// removedSysctlNames returns the sysctls a scope currently manages that the next
+// stored map no longer will, so they can be warned about before the write happens.
+func removedSysctlNames(property string, next map[string]string) ([]string, error) {
+	current, err := common.PropertyMapGet("scheduler-k3s", "--global", property)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to get property map: %w", err)
+	}
+
+	removed := []string{}
+	for key := range current {
+		if _, ok := next[key]; !ok {
+			removed = append(removed, key)
+		}
+	}
+
+	sort.Strings(removed)
+	return removed, nil
+}
+
+// warnUnmanagedSysctls notes that dokku has stopped managing a sysctl. The last
+// value written stays on affected nodes until they reboot, which is how sysctl -w
+// behaves everywhere else.
+func warnUnmanagedSysctls(names []string) {
+	for _, name := range names {
+		common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", name))
+	}
+}
+
 // CommandNodeSysctlsReport displays the configured node-level kernel sysctls
 func CommandNodeSysctlsReport(format string) error {
 	if format != "stdout" && format != "json" {
@@ -149,8 +225,8 @@ func verifyNodeProfileSysctlsSupported(profileName string) error {
 	return nil
 }
 
-// CommandAutoscalingAuthSet set or clear a scheduler-k3s autoscaling keda trigger authentication object for an app
+// CommandAutoscalingAuthSet sets, clears or replaces a scheduler-k3s autoscaling keda trigger authentication object for an app
-func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool) error {
+func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool, replace bool) error {
 	if global {
 		appName = "--global"
 	}
@@ -165,18 +241,17 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri
 		return fmt.Errorf("Missing trigger type argument")
 	}
 
+	if replace && len(metadata) == 0 {
+		return fmt.Errorf("Must specify at least one --metadata flag, omit --replace to remove all metadata for the trigger")
+	}
+
+	if len(metadata) == 0 || replace {
+		if err := deleteTriggerAuthProperties(appName, trigger); err != nil {
+			return err
+		}
+	}
+
 	if len(metadata) == 0 {
-		properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, fmt.Sprintf("%s%s.", TriggerAuthPropertyPrefix, trigger))
-		if err != nil {
-			return fmt.Errorf("Unable to get property list: %w", err)
-		}
-
-		for key := range properties {
-			if err := common.PropertyDelete("scheduler-k3s", appName, key); err != nil {
-				return fmt.Errorf("Unable to delete property: %w", err)
-			}
-		}
-
 		if appName == "--global" {
 			helmAgent, err := NewHelmAgent("keda", DeployLogPrinter)
 			if err != nil {
@@ -196,11 +271,11 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri
 		if err := common.PropertyWrite("scheduler-k3s", appName, fmt.Sprintf("%s%s.%s", TriggerAuthPropertyPrefix, trigger, key), value); err != nil {
 			return fmt.Errorf("Unable to set property: %w", err)
 		}
-
-		common.LogInfo1("Trigger authentication settings saved")
-		common.LogVerbose("Resources will be created or updated on next deploy")
 	}
 
+	common.LogInfo1("Trigger authentication settings saved")
+	common.LogVerbose("Resources will be created or updated on next deploy")
+
 	if appName == "--global" {
 		err := applyKedaClusterTriggerAuthentications(context.Background(), trigger, metadata)
 		if err != nil {
@@ -211,6 +286,22 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri
 	return nil
 }
 
+// deleteTriggerAuthProperties removes every stored metadata key for one trigger.
+func deleteTriggerAuthProperties(appName string, trigger string) error {
+	properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, fmt.Sprintf("%s%s.", TriggerAuthPropertyPrefix, trigger))
+	if err != nil {
+		return fmt.Errorf("Unable to get property list: %w", err)
+	}
+
+	for key := range properties {
+		if err := common.PropertyDelete("scheduler-k3s", appName, key); err != nil {
+			return fmt.Errorf("Unable to delete property: %w", err)
+		}
+	}
+
+	return nil
+}
+
 // CommandAutoscalingAuthReport displays a scheduler-k3s autoscaling keda trigger authentication report for one or more apps
 func CommandAutoscalingAuthReport(appName string, format string, includeMetadata bool, infoFlag string) error {
 	if len(appName) == 0 {
@@ -1237,29 +1328,14 @@ func CommandEnsureCharts(forceInstall bool, forceChartNames []string) error {
 	return nil
 }
 
-// CommandLabelsSet set or clear a scheduler-k3s label for an app
+// CommandLabelsSet sets, clears or replaces scheduler-k3s labels for an app
-func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error {
+func CommandLabelsSet(input MetadataSetInput) error {
-	if resourceType == "" {
+	return commandMetadataSet(labelsField, input)
-		return fmt.Errorf("Missing resource-type")
+}
-	}
 
-	if processType == "" {
+// CommandLabelsClear clears scheduler-k3s labels for an app
-		processType = GlobalProcessType
+func CommandLabelsClear(input MetadataClearInput) error {
-	}
+	return commandMetadataClear(labelsField, input)
-
-	property := fmt.Sprintf("labels.%s.%s", processType, resourceType)
-	if value == "" {
-		if err := common.PropertyMapDelete("scheduler-k3s", appName, property, key); err != nil {
-			return fmt.Errorf("Unable to delete property map entry: %w", err)
-		}
-		return nil
-	}
-
-	if err := common.PropertyMapSet("scheduler-k3s", appName, property, key, value); err != nil {
-		return fmt.Errorf("Unable to set property map entry: %w", err)
-	}
-
-	return nil
 }
 
 // CommandProfilesAdd adds a node profile to the k3s cluster

tests/unit/scheduler-k3s-annotations.bats

@@ -10,10 +10,8 @@ setup() {
 teardown() {
   dokku --force apps:destroy $TEST_APP >/dev/null 2>/dev/null || true
   dokku --force apps:destroy ${TEST_APP}-2 >/dev/null 2>/dev/null || true
-  for resource in deployment service pod cronjob job secret ingress serviceaccount; do
+  dokku scheduler-k3s:annotations:clear --global >/dev/null 2>/dev/null || true
-    dokku scheduler-k3s:annotations:set --global --resource-type "$resource" foo "" >/dev/null 2>/dev/null || true
+  dokku scheduler-k3s:node-sysctls:clear --global >/dev/null 2>/dev/null || true
-    dokku scheduler-k3s:annotations:set --global --resource-type "$resource" bar "" >/dev/null 2>/dev/null || true
-  done
   global_teardown
 }
 
@@ -113,3 +111,166 @@ teardown() {
   assert_failure
   assert_output_contains "Invalid flag passed"
 }
+
+@test "(scheduler-k3s:annotations:set) --replace writes the declared map in one call" {
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment keep old-value"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment drop dropped-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment keep=new-value add=added-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.add,global.deployment.keep"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r '.\"global.deployment.keep\"'"
+  assert_success
+  assert_output "new-value"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment --process-type web scoped=scoped-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.add,global.deployment.keep,web.deployment.scoped"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace --global --resource-type deployment global-key=global-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report --global --format json | jq -r '.\"global.deployment.global-key\"'"
+  assert_success
+  assert_output "global-value"
+}
+
+@test "(scheduler-k3s:annotations:set) --replace stores empty values and rejects malformed pair lists" {
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment empty="
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r '.\"global.deployment.empty\"'"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment"
+  assert_failure
+  assert_output_contains "Must specify at least one key=value pair, use scheduler-k3s:annotations:clear to remove all annotations"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment nodelimiter"
+  assert_failure
+  assert_output_contains "Invalid key=value pair: nodelimiter"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment foo=bar foo=qux"
+  assert_failure
+  assert_output_contains "Duplicate key specified: foo"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.empty"
+}
+
+@test "(scheduler-k3s:annotations:set) rejects an unknown resource-type and a missing app" {
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deploymnet foo bar"
+  assert_failure
+  assert_output_contains "Invalid resource-type specified, valid resource types include:"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deploymnet foo=bar"
+  assert_failure
+  assert_output_contains "Invalid resource-type specified, valid resource types include:"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set ${TEST_APP}-missing --resource-type deployment foo bar"
+  assert_failure
+  assert_output_contains "App ${TEST_APP}-missing does not exist"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace ${TEST_APP}-missing --resource-type deployment foo=bar"
+  assert_failure
+  assert_output_contains "App ${TEST_APP}-missing does not exist"
+}
+
+@test "(scheduler-k3s:annotations:clear) removes annotations, honouring filters" {
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment foo bar"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --process-type web --resource-type deployment baz qux"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type service alpha beta"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP --process-type web"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.foo,global.service.alpha"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP --resource-type service"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.foo"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json"
+  assert_success
+  assert_output "{}"
+}
+
+@test "(scheduler-k3s:annotations:clear) leaves neighbouring scheduler-k3s properties alone" {
+  run /bin/bash -c "dokku scheduler-k3s:set $TEST_APP deploy-timeout 300s"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-1"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment foo bar"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json"
+  assert_success
+  assert_output "{}"
+
+  run /bin/bash -c "dokku --quiet scheduler-k3s:report $TEST_APP --scheduler-k3s-deploy-timeout"
+  assert_success
+  assert_output "300s"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\".\"vm.max_map_count\"'"
+  assert_success
+  assert_output "262144"
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r '.\"datadog.apiKey\"'"
+  assert_success
+  assert_output "secret-1"
+}
+
+@test "(scheduler-k3s:annotations:clear) supports --global and requires an app" {
+  run /bin/bash -c "dokku scheduler-k3s:annotations:set --global --resource-type deployment foo global-val"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear"
+  assert_failure
+  assert_output_contains "Please specify an app to run the command on"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear ${TEST_APP}-missing"
+  assert_failure
+  assert_output_contains "App ${TEST_APP}-missing does not exist"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP --resource-type deploymnet"
+  assert_failure
+  assert_output_contains "Invalid resource-type specified, valid resource types include:"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report --global --format json | jq -r '.\"global.deployment.foo\"'"
+  assert_success
+  assert_output "global-val"
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:clear --global"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:annotations:report --global --format json"
+  assert_success
+  assert_output "{}"
+}

tests/unit/scheduler-k3s-autoscaling-auth.bats

@@ -149,3 +149,44 @@ teardown() {
   assert_failure
   assert_output_contains "Invalid flag passed"
 }
+
+@test "(scheduler-k3s:autoscaling-auth:set) --replace drops metadata keys not named" {
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-1 --metadata appKey=secret-2 --metadata datadogSite=us5.datadoghq.com"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-3"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "datadog.apiKey,datadog.appKey,datadog.datadogSite"
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set --replace $TEST_APP datadog --metadata apiKey=secret-4"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "datadog.apiKey"
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r '.\"datadog.apiKey\"'"
+  assert_success
+  assert_output "secret-4"
+}
+
+@test "(scheduler-k3s:autoscaling-auth:set) --replace requires metadata and leaves other triggers alone" {
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-1"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP memory --metadata some-key=some-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set --replace $TEST_APP datadog"
+  assert_failure
+  assert_output_contains "Must specify at least one --metadata flag, omit --replace to remove all metadata for the trigger"
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set --replace $TEST_APP datadog --metadata appKey=secret-2"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "datadog.appKey,memory.some-key"
+}

tests/unit/scheduler-k3s-labels.bats

@@ -10,10 +10,7 @@ setup() {
 teardown() {
   dokku --force apps:destroy $TEST_APP >/dev/null 2>/dev/null || true
   dokku --force apps:destroy ${TEST_APP}-2 >/dev/null 2>/dev/null || true
-  for resource in deployment service pod cronjob job secret ingress serviceaccount; do
+  dokku scheduler-k3s:labels:clear --global >/dev/null 2>/dev/null || true
-    dokku scheduler-k3s:labels:set --global --resource-type "$resource" foo "" >/dev/null 2>/dev/null || true
-    dokku scheduler-k3s:labels:set --global --resource-type "$resource" bar "" >/dev/null 2>/dev/null || true
-  done
   global_teardown
 }
 
@@ -113,3 +110,136 @@ teardown() {
   assert_failure
   assert_output_contains "Invalid flag passed"
 }
+
+@test "(scheduler-k3s:labels:set) --replace writes the declared map in one call" {
+  run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deployment keep old-value"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deployment drop dropped-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment keep=new-value add=added-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.add,global.deployment.keep"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r '.\"global.deployment.keep\"'"
+  assert_success
+  assert_output "new-value"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment --process-type web scoped=scoped-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.add,global.deployment.keep,web.deployment.scoped"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace --global --resource-type deployment global-key=global-value"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report --global --format json | jq -r '.\"global.deployment.global-key\"'"
+  assert_success
+  assert_output "global-value"
+}
+
+@test "(scheduler-k3s:labels:set) --replace stores empty values and rejects malformed pair lists" {
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment empty="
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r '.\"global.deployment.empty\"'"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment"
+  assert_failure
+  assert_output_contains "Must specify at least one key=value pair, use scheduler-k3s:labels:clear to remove all labels"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment nodelimiter"
+  assert_failure
+  assert_output_contains "Invalid key=value pair: nodelimiter"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment foo=bar foo=qux"
+  assert_failure
+  assert_output_contains "Duplicate key specified: foo"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.empty"
+}
+
+@test "(scheduler-k3s:labels:set) rejects an unknown resource-type and a missing app" {
+  run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deploymnet foo bar"
+  assert_failure
+  assert_output_contains "Invalid resource-type specified, valid resource types include:"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deploymnet foo=bar"
+  assert_failure
+  assert_output_contains "Invalid resource-type specified, valid resource types include:"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set ${TEST_APP}-missing --resource-type deployment foo bar"
+  assert_failure
+  assert_output_contains "App ${TEST_APP}-missing does not exist"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --replace ${TEST_APP}-missing --resource-type deployment foo=bar"
+  assert_failure
+  assert_output_contains "App ${TEST_APP}-missing does not exist"
+}
+
+@test "(scheduler-k3s:labels:clear) removes labels, honouring filters" {
+  run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deployment foo bar"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --process-type web --resource-type deployment baz qux"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type service alpha beta"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP --process-type web"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.foo,global.service.alpha"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP --resource-type service"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'"
+  assert_success
+  assert_output "global.deployment.foo"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json"
+  assert_success
+  assert_output "{}"
+}
+
+@test "(scheduler-k3s:labels:clear) supports --global and requires an app" {
+  run /bin/bash -c "dokku scheduler-k3s:labels:set --global --resource-type deployment foo global-val"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:clear"
+  assert_failure
+  assert_output_contains "Please specify an app to run the command on"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:clear ${TEST_APP}-missing"
+  assert_failure
+  assert_output_contains "App ${TEST_APP}-missing does not exist"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP --resource-type deploymnet"
+  assert_failure
+  assert_output_contains "Invalid resource-type specified, valid resource types include:"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report --global --format json | jq -r '.\"global.deployment.foo\"'"
+  assert_success
+  assert_output "global-val"
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:clear --global"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:labels:report --global --format json"
+  assert_success
+  assert_output "{}"
+}

tests/unit/scheduler-k3s-node-sysctls.bats

@@ -0,0 +1,87 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+setup() {
+  global_setup
+}
+
+teardown() {
+  dokku scheduler-k3s:node-sysctls:clear --global >/dev/null 2>/dev/null || true
+  dokku scheduler-k3s:node-sysctls:clear --profile edge-workers >/dev/null 2>/dev/null || true
+  dokku scheduler-k3s:profiles:remove edge-workers >/dev/null 2>/dev/null || true
+  global_teardown
+}
+
+@test "(scheduler-k3s:node-sysctls:set) --replace writes the declared map in one call" {
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144"
+  assert_success
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.swappiness 10"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.swappiness=20 vm.overcommit_memory=1"
+  assert_success
+  assert_output_contains "Removing vm.max_map_count stops dokku managing it"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\" | keys | sort | join(\",\")'"
+  assert_success
+  assert_output "vm.overcommit_memory,vm.swappiness"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\".\"vm.swappiness\"'"
+  assert_success
+  assert_output "20"
+
+  run /bin/bash -c "dokku scheduler-k3s:profiles:add edge-workers --role worker"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --profile edge-workers vm.swappiness=60"
+  assert_success
+
+  # a profile scope inherits the global sysctls and overrides them on conflict
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"edge-workers\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'"
+  assert_success
+  assert_output "vm.overcommit_memory=1,vm.swappiness=60"
+}
+
+@test "(scheduler-k3s:node-sysctls:set) --replace rejects malformed pair lists without writing" {
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global"
+  assert_failure
+  assert_output_contains "Must specify at least one key=value pair, use scheduler-k3s:node-sysctls:clear to remove all sysctls"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global nodelimiter"
+  assert_failure
+  assert_output_contains "Invalid key=value pair: nodelimiter"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.swappiness=10 vm.swappiness=20"
+  assert_failure
+  assert_output_contains "Duplicate key specified: vm.swappiness"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\" | keys | join(\",\")'"
+  assert_success
+  assert_output "vm.max_map_count"
+}
+
+@test "(scheduler-k3s:node-sysctls:clear) removes every sysctl for a scope" {
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.max_map_count=262144 vm.swappiness=10"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --global --profile edge-workers"
+  assert_failure
+  assert_output_contains "Only one of --global and --profile may be specified"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --profile missing-profile"
+  assert_failure
+  assert_output_contains "Node profile missing-profile not found"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --global"
+  assert_success
+  assert_output_contains "Removing vm.max_map_count stops dokku managing it"
+  assert_output_contains "Removing vm.swappiness stops dokku managing it"
+
+  run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\" | length'"
+  assert_success
+  assert_output "0"
+}