From c37a71b90ba53766a8a711613e92aa378e614c44 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 18 Sep 2026 05:14:51 -0400 Subject: [PATCH] feat: add a whole-set form for scheduler-k3s metadata The `scheduler-k3s` metadata commands were all per-key, so matching an app's stored annotations, labels or node sysctls to a declared set meant reading the report, computing the difference in both directions and issuing one command per added, changed and removed key, with a failure partway through leaving a mixture of the two sets. A `--replace` flag on `annotations:set`, `labels:set`, `node-sysctls:set` and `autoscaling-auth:set` now writes the declared map in a single call, taking trailing `key=value` pairs everywhere except `autoscaling-auth:set`, which keeps its repeatable `--metadata` flag. Every pair is parsed before anything is written, so a rejected pair leaves the stored map untouched, and a pair ending in `=` stores an empty value, which the per-key form cannot express. An empty pair list is rejected rather than treated as a request to remove everything, so a generated list that expands to nothing cannot silently drop an app's metadata; `annotations:clear`, `labels:clear` and `node-sysctls:clear` handle that case, taking the same optional filters as their report commands. The `--resource-type` flag and the app name are now validated on the set commands as well, where a typo previously exited zero and wrote a property that no report and no deploy ever read. --- docs/deployment/schedulers/k3s.md | 106 ++++++- plugins/scheduler-k3s/Makefile | 2 +- plugins/scheduler-k3s/metadata.go | 275 ++++++++++++++++++ plugins/scheduler-k3s/metadata_test.go | 188 ++++++++++++ plugins/scheduler-k3s/report.go | 97 +----- .../scheduler-k3s/src/commands/commands.go | 7 + .../src/subcommands/subcommands.go | 122 ++++++-- plugins/scheduler-k3s/subcommands.go | 234 ++++++++++----- tests/unit/scheduler-k3s-annotations.bats | 169 ++++++++++- .../unit/scheduler-k3s-autoscaling-auth.bats | 41 +++ tests/unit/scheduler-k3s-labels.bats | 138 ++++++++- tests/unit/scheduler-k3s-node-sysctls.bats | 87 ++++++ 12 files changed, 1271 insertions(+), 195 deletions(-) create mode 100644 plugins/scheduler-k3s/metadata.go create mode 100644 plugins/scheduler-k3s/metadata_test.go create mode 100644 tests/unit/scheduler-k3s-node-sysctls.bats diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 6f3489b4e..00cd536dc 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -4,9 +4,12 @@ > New as of 0.33.0 ``` +scheduler-k3s:annotations:clear [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Clear all annotations for an app or a single process-type/resource-type scope scheduler-k3s:annotations:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Set or clear an annotation for a given app/process-type/resource-type combination +scheduler-k3s:annotations:set --replace [ ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Replace the entire annotation map for a given app/process-type/resource-type combination scheduler-k3s:annotations:report [|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Displays a scheduler-k3s annotations report for one or more apps scheduler-k3s:autoscaling-auth:set [<--metadata key=value>...] # Set or clear a scheduler-k3s autoscaling keda trigger authentication resource for an app +scheduler-k3s:autoscaling-auth:set --replace <--metadata key=value> [<--metadata key=value>...] # Replace the entire metadata map for a scheduler-k3s autoscaling keda trigger authentication resource scheduler-k3s:autoscaling-auth:report [|--global] [--format stdout|json] [--include-metadata] # Displays a scheduler-k3s autoscaling auth report for one or more apps scheduler-k3s:charts:report [] [--format stdout|json] # Displays a scheduler-k3s chart override report scheduler-k3s:charts:set () # Set or clear a chart-specific helm value @@ -15,9 +18,13 @@ scheduler-k3s:cluster:list [--format json|stdout] # Lists all nodes in a Dokku-m scheduler-k3s:cluster:remove [node-id] # Removes client node to a Dokku-managed cluster scheduler-k3s:ensure-charts # Ensures the k3s charts are installed scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] # Initializes a cluster +scheduler-k3s:labels:clear [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Clear all labels for an app or a single process-type/resource-type scope scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Set or clear a label for a given app/process-type/resource-type combination +scheduler-k3s:labels:set --replace [ ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Replace the entire label map for a given app/process-type/resource-type combination scheduler-k3s:labels:report [|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Displays a scheduler-k3s labels report for one or more apps +scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE] # Clear all node-level kernel sysctls for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:set () [--global|--profile PROFILE] # Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile +scheduler-k3s:node-sysctls:set --replace [ ...] [--global|--profile PROFILE] # Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:report [--format stdout|json] # Displays the node-level kernel sysctls applied to each scope scheduler-k3s:preview [--context N] [--show-secrets] [--show-secrets-decoded] # Displays a diff between the current and next deployment for an app scheduler-k3s:profiles:add [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] # Adds a node profile to the k3s cluster @@ -595,6 +602,49 @@ dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type d A `ps:restart` is required after removing annotations in order to remove them from running resources. +#### Replacing the entire annotation set + +The per-key form above changes one annotation at a time, so matching the stored annotations to a declared set means computing the difference and issuing one command per added, changed and removed key. The `--replace` flag writes the whole map for a scope in a single call, taking `key=value` pairs instead of a single key and value. + +```shell +dokku scheduler-k3s:annotations:set --replace node-js-app --resource-type deployment annotation.key=annotation.value other.key=other.value +``` + +Anything previously stored for that scope and not named in the call is removed. `--resource-type` is still required, and `--process-type` scopes the replacement the same way it scopes a single key. + +```shell +dokku scheduler-k3s:annotations:set --replace node-js-app --resource-type deployment --process-type web annotation.key=annotation.value +``` + +Every pair is parsed before anything is written, so a rejected pair leaves the stored annotations untouched. A pair without a `=`, a pair with an empty key, and a key named more than once are all rejected. + +A pair ending in `=` stores an empty value, which the per-key form cannot express - there, an empty value deletes the key instead. + +```shell +dokku scheduler-k3s:annotations:set --replace node-js-app --resource-type deployment annotation.key= +``` + +> [!NOTE] +> An empty pair list is rejected rather than treated as a request to remove everything, so a generated list that expands to nothing cannot silently drop an app's annotations. Use `scheduler-k3s:annotations:clear` for that. + +#### Clearing annotations + +The `scheduler-k3s:annotations:clear` command removes every annotation configured for an app. + +```shell +dokku scheduler-k3s:annotations:clear node-js-app +``` + +The `--process-type` and `--resource-type` flags narrow what is cleared, and filter the same way they do for `scheduler-k3s:annotations:report`. Omitting both clears every scope. + +```shell +dokku scheduler-k3s:annotations:clear node-js-app --resource-type deployment +dokku scheduler-k3s:annotations:clear node-js-app --process-type web --resource-type deployment +dokku scheduler-k3s:annotations:clear --global --resource-type deployment +``` + +A `ps:restart` is required after clearing annotations in order to remove them from running resources. + #### Displaying annotations Configured annotations can be inspected with the `scheduler-k3s:annotations:report` command. Without arguments, it iterates every app and prints all annotations. Passing an app name (or `--global`) scopes the report: @@ -658,15 +708,38 @@ A `ps:restart` is required after setting labels in order to have them apply to r #### Removing a label -To unset an label, pass an empty value: +To unset a label, pass an empty value: ```shell -dokku scheduler-k3s:annotations:set node-js-app label.key --resource-type deployment +dokku scheduler-k3s:labels:set node-js-app label.key --resource-type deployment dokku scheduler-k3s:labels:set node-js-app label.key --resource-type deployment --process-type web ``` A `ps:restart` is required after removing labels in order to remove them from running resources. +#### Replacing the entire label set + +As with annotations, the `--replace` flag writes the whole map for a scope in a single call, taking `key=value` pairs instead of a single key and value. + +```shell +dokku scheduler-k3s:labels:set --replace node-js-app --resource-type deployment label.key=label.value other.key=other.value +``` + +Anything previously stored for that scope and not named in the call is removed. `--resource-type` is still required, `--process-type` scopes the replacement, every pair is parsed before anything is written, and a pair ending in `=` stores an empty value. An empty pair list is rejected; use `scheduler-k3s:labels:clear` to remove everything. + +#### Clearing labels + +The `scheduler-k3s:labels:clear` command removes every label configured for an app, with the same optional `--process-type` and `--resource-type` filters as `scheduler-k3s:labels:report`. + +```shell +dokku scheduler-k3s:labels:clear node-js-app +dokku scheduler-k3s:labels:clear node-js-app --resource-type deployment +dokku scheduler-k3s:labels:clear node-js-app --process-type web --resource-type deployment +dokku scheduler-k3s:labels:clear --global --resource-type deployment +``` + +A `ps:restart` is required after clearing labels in order to remove them from running resources. + #### Displaying labels Configured labels can be inspected with the `scheduler-k3s:labels:report` command. The surface mirrors `scheduler-k3s:annotations:report`: @@ -809,9 +882,19 @@ After execution, Dokku will include the following resources for each specified t If the `--global` flag is specified instead of an app name, a custom helm chart is created on the fly with the above resources. +##### Replacing Authentication Metadata + +Without `--replace`, the metadata supplied is merged into whatever is already stored for the trigger, so a key that is no longer named stays behind. The `--replace` flag drops the trigger's existing metadata before writing, making the stored set exactly what was specified. + +```shell +dokku scheduler-k3s:autoscaling-auth:set --replace node-js-app datadog --metadata apiKey=1234567890 --metadata appKey=asdfghjkl +``` + +In the example above, a `datadogSite` key configured earlier is removed rather than left in place. At least one `--metadata` flag is required with `--replace`; omit the flag entirely to remove everything, as below. + ##### Removing Authentication Resources -To remove a configured authenticatin resource, run the `scheduler-k3s:autoscaling-auth:set` command with no metadata specified. Subsequent deploys will not include these resources. +To remove a configured authentication resource, run the `scheduler-k3s:autoscaling-auth:set` command with no metadata specified. Subsequent deploys will not include these resources. ```shell dokku scheduler-k3s:autoscaling-auth:set $APP $TRIGGER_TYPE @@ -908,6 +991,23 @@ A profile scope inherits everything set globally and overrides it on conflict, s A profile created by an older Dokku may carry a name outside the [current rules](#adding-profiles), such as one containing uppercase or longer than 26 characters. Dokku cannot name a helm release after such a profile, so it is skipped with a warning and `node-sysctls:set --profile` refuses it. Recreate the profile under a valid name to give it sysctls; `scheduler-k3s:profiles:remove` still removes the old one. +The `--replace` flag writes the whole set for a scope in one call, taking `sysctl=value` pairs instead of a single name and value. Anything previously set for that scope and not named in the call is cleared, with the same caveat about the value staying on affected nodes until they reboot. + +```shell +dokku scheduler-k3s:node-sysctls:set --replace --global vm.max_map_count=262144 vm.swappiness=10 +``` + +Every pair is parsed before anything is written, so a rejected pair leaves the stored sysctls untouched. A pair without a `=`, a pair with an empty key, and a key named more than once are all rejected. An empty pair list is rejected rather than treated as a request to remove everything. + +To remove every sysctl for a scope, use `scheduler-k3s:node-sysctls:clear`. + +```shell +dokku scheduler-k3s:node-sysctls:clear --global +dokku scheduler-k3s:node-sysctls:clear --profile edge-workers +``` + +Clearing a scope removes its DaemonSet. As with clearing a single sysctl, the values it last wrote stay in place on affected nodes until they reboot. + Use `node-sysctls:report` to see the resolved set for every scope. ```shell diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index 340975586..d175aa3ad 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,4 +1,4 @@ -SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall +SUBCOMMANDS = subcommands/annotations:clear subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:clear subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:clear subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/scheduler-uses-host-cron triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/metadata.go b/plugins/scheduler-k3s/metadata.go new file mode 100644 index 000000000..6db2fd577 --- /dev/null +++ b/plugins/scheduler-k3s/metadata.go @@ -0,0 +1,275 @@ +package scheduler_k3s + +import ( + "fmt" + "sort" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +// metadataField selects which of the scheduler-k3s metadata maps a command +// operates on, so the annotations and labels commands can share one +// implementation the way the storage plugin does. +type metadataField struct { + // Name is the plural noun used in user-facing messages. + Name string + // Singular is the singular noun used in user-facing messages. + Singular string + // ClearCommand is the command an empty --replace list points the user at. + ClearCommand string + // PropertyPrefix namespaces the property names this field owns. Annotations + // are stored unprefixed, which is why annotations alone must skip the + // property names reserved by other scheduler-k3s features. + PropertyPrefix string + // SkipReserved drops property names owned by other scheduler-k3s features. + SkipReserved bool + // ResourceTypes lists the kubernetes resource types this field supports. + ResourceTypes []string +} + +var annotationsField = metadataField{ + Name: "annotations", + Singular: "annotation", + ClearCommand: "scheduler-k3s:annotations:clear", + SkipReserved: true, + ResourceTypes: AnnotationResourceTypes, +} + +var labelsField = metadataField{ + Name: "labels", + Singular: "label", + ClearCommand: "scheduler-k3s:labels:clear", + PropertyPrefix: "labels.", + ResourceTypes: LabelResourceTypes, +} + +// MetadataSetInput captures the inputs accepted by annotations:set and labels:set. +// Key/Value and Pairs are mutually exclusive: Pairs is populated only under +// --replace, where the positional arguments are key=value pairs rather than a +// single key and value. +type MetadataSetInput struct { + AppName string + ProcessType string + ResourceType string + Key string + Value string + Pairs []string + Replace bool +} + +// MetadataClearInput captures the inputs accepted by annotations:clear and labels:clear. +// An empty ProcessType or ResourceType filters nothing, matching the report commands. +type MetadataClearInput struct { + AppName string + ProcessType string + ResourceType string +} + +// metadataPropertyName returns the property a field stores one scope's map under. +func metadataPropertyName(field metadataField, processType string, resourceType string) string { + return fmt.Sprintf("%s%s.%s", field.PropertyPrefix, processType, resourceType) +} + +// verifyMetadataAppName applies the standard app name check, exempting the global scope. +func verifyMetadataAppName(appName string) error { + if appName == "--global" { + return nil + } + + return common.VerifyAppName(appName) +} + +// validateResourceType reports whether a resource type is one the field supports. +// An empty resource type passes, as the clear commands treat it as an absent filter; +// callers that require one check for it separately. +func validateResourceType(field metadataField, resourceType string) error { + if resourceType == "" { + return nil + } + + for _, valid := range field.ResourceTypes { + if valid == resourceType { + return nil + } + } + + valid := make([]string, len(field.ResourceTypes)) + copy(valid, field.ResourceTypes) + sort.Strings(valid) + return fmt.Errorf("Invalid resource-type specified, valid resource types include: %s", strings.Join(valid, ", ")) +} + +// parseMetadataPairs turns key=value arguments into a map. A pair without a '=' +// or with an empty key is rejected, as is a key specified more than once, since +// a repeated key leaves the declared map ambiguous. A pair ending in '=' stores +// an empty value, which the per-key form cannot express. +func parseMetadataPairs(pairs []string) (map[string]string, error) { + parsed := map[string]string{} + for _, pair := range pairs { + parts := strings.SplitN(pair, "=", 2) + if len(parts) != 2 || parts[0] == "" { + return nil, fmt.Errorf("Invalid key=value pair: %s", pair) + } + + if _, ok := parsed[parts[0]]; ok { + return nil, fmt.Errorf("Duplicate key specified: %s", parts[0]) + } + + parsed[parts[0]] = parts[1] + } + + return parsed, nil +} + +// setMetadataKey writes a single key on one of the metadata maps. An empty value +// deletes just that key, leaving its siblings in place. +func setMetadataKey(field metadataField, input MetadataSetInput) error { + property := metadataPropertyName(field, input.ProcessType, input.ResourceType) + if input.Value == "" { + if err := common.PropertyMapDelete("scheduler-k3s", input.AppName, property, input.Key); err != nil { + return fmt.Errorf("Unable to delete property map entry: %w", err) + } + return nil + } + + if err := common.PropertyMapSet("scheduler-k3s", input.AppName, property, input.Key, input.Value); err != nil { + return fmt.Errorf("Unable to set property map entry: %w", err) + } + + return nil +} + +// replaceMetadata swaps one scope's whole map for the declared one in a single write. +// Every pair is parsed before anything is written, so a rejected pair leaves the +// stored map untouched. +func replaceMetadata(field metadataField, input MetadataSetInput) error { + if len(input.Pairs) == 0 { + return fmt.Errorf("Must specify at least one key=value pair, use %s to remove all %s", field.ClearCommand, field.Name) + } + + parsed, err := parseMetadataPairs(input.Pairs) + if err != nil { + return err + } + + property := metadataPropertyName(field, input.ProcessType, input.ResourceType) + if err := common.PropertyMapWrite("scheduler-k3s", input.AppName, property, parsed); err != nil { + return fmt.Errorf("Unable to write property map: %w", err) + } + + return nil +} + +// clearMetadata removes every metadata property matching the given filters. +func clearMetadata(field metadataField, input MetadataClearInput) error { + properties, err := matchingMetadataProperties(field, input.AppName, input.ProcessType, input.ResourceType) + if err != nil { + return err + } + + for _, property := range properties { + if err := common.PropertyDelete("scheduler-k3s", input.AppName, property.PropertyName); err != nil { + return fmt.Errorf("Unable to delete property: %w", err) + } + } + + return nil +} + +// metadataProperty is one stored metadata scope, as found by scanning the property store. +type metadataProperty struct { + // PropertyName is the name the scope is stored under, prefix included. + PropertyName string + ProcessType string + ResourceType string +} + +// matchingMetadataProperties scans the property store for the scopes a field owns on +// appName, optionally filtered by processType/resourceType. Both the report commands +// and the clear commands read through this, so they cannot disagree about what is in scope. +func matchingMetadataProperties(field metadataField, appName string, processType string, resourceType string) ([]metadataProperty, error) { + properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, field.PropertyPrefix) + if err != nil { + return nil, fmt.Errorf("Unable to get property list: %w", err) + } + + knownResourceTypes := map[string]bool{} + for _, rt := range field.ResourceTypes { + knownResourceTypes[rt] = true + } + + matching := []metadataProperty{} + for propertyName := range properties { + if field.SkipReserved && isReservedAnnotationProperty(propertyName) { + continue + } + + suffix := strings.TrimPrefix(propertyName, field.PropertyPrefix) + dot := strings.LastIndex(suffix, ".") + if dot <= 0 || dot == len(suffix)-1 { + continue + } + + propProcessType := suffix[:dot] + propResourceType := suffix[dot+1:] + if !knownResourceTypes[propResourceType] { + continue + } + + if processType != "" && propProcessType != processType { + continue + } + if resourceType != "" && propResourceType != resourceType { + continue + } + + matching = append(matching, metadataProperty{ + PropertyName: propertyName, + ProcessType: propProcessType, + ResourceType: propResourceType, + }) + } + + return matching, nil +} + +// commandMetadataSet implements the annotations:set and labels:set commands. +func commandMetadataSet(field metadataField, input MetadataSetInput) error { + if err := verifyMetadataAppName(input.AppName); err != nil { + return err + } + + if input.ResourceType == "" { + return fmt.Errorf("Missing resource-type") + } + + if err := validateResourceType(field, input.ResourceType); err != nil { + return err + } + + if input.ProcessType == "" { + input.ProcessType = GlobalProcessType + } + + if input.Replace { + return replaceMetadata(field, input) + } + + return setMetadataKey(field, input) +} + +// commandMetadataClear implements the annotations:clear and labels:clear commands. +// Unlike the set commands, an omitted process-type or resource-type is an absent +// filter rather than a default scope, matching the report commands. +func commandMetadataClear(field metadataField, input MetadataClearInput) error { + if err := verifyMetadataAppName(input.AppName); err != nil { + return err + } + + if err := validateResourceType(field, input.ResourceType); err != nil { + return err + } + + return clearMetadata(field, input) +} diff --git a/plugins/scheduler-k3s/metadata_test.go b/plugins/scheduler-k3s/metadata_test.go new file mode 100644 index 000000000..ccfea0e41 --- /dev/null +++ b/plugins/scheduler-k3s/metadata_test.go @@ -0,0 +1,188 @@ +package scheduler_k3s + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseMetadataPairs(t *testing.T) { + tests := []struct { + name string + pairs []string + want map[string]string + }{ + { + name: "empty list", + pairs: []string{}, + want: map[string]string{}, + }, + { + name: "multiple pairs", + pairs: []string{"foo=bar", "baz=qux"}, + want: map[string]string{"foo": "bar", "baz": "qux"}, + }, + { + name: "trailing equals stores an empty value", + pairs: []string{"foo="}, + want: map[string]string{"foo": ""}, + }, + { + name: "value containing equals is kept whole", + pairs: []string{"foo=a=b"}, + want: map[string]string{"foo": "a=b"}, + }, + { + name: "kubernetes style key", + pairs: []string{"prometheus.io/scrape=true"}, + want: map[string]string{"prometheus.io/scrape": "true"}, + }, + { + name: "multi-line value", + pairs: []string{"foo=line one\nline two"}, + want: map[string]string{"foo": "line one\nline two"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := parseMetadataPairs(test.pairs) + if err != nil { + t.Fatalf("parseMetadataPairs(%q) returned error: %s", test.pairs, err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("parseMetadataPairs(%q) = %v, want %v", test.pairs, got, test.want) + } + }) + } +} + +func TestParseMetadataPairsErrors(t *testing.T) { + tests := []struct { + name string + pairs []string + want string + }{ + { + name: "missing equals", + pairs: []string{"foo"}, + want: "Invalid key=value pair: foo", + }, + { + name: "empty key", + pairs: []string{"=bar"}, + want: "Invalid key=value pair: =bar", + }, + { + name: "empty pair", + pairs: []string{""}, + want: "Invalid key=value pair: ", + }, + { + name: "duplicate key", + pairs: []string{"foo=bar", "foo=qux"}, + want: "Duplicate key specified: foo", + }, + { + name: "duplicate key with identical value", + pairs: []string{"foo=bar", "foo=bar"}, + want: "Duplicate key specified: foo", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := parseMetadataPairs(test.pairs) + if err == nil { + t.Fatalf("parseMetadataPairs(%q) = %v, want error", test.pairs, got) + } + if err.Error() != test.want { + t.Errorf("parseMetadataPairs(%q) error = %q, want %q", test.pairs, err.Error(), test.want) + } + }) + } +} + +func TestValidateResourceType(t *testing.T) { + tests := []struct { + name string + field metadataField + resourceType string + wantErr bool + }{ + { + name: "valid annotation resource type", + field: annotationsField, + resourceType: "deployment", + }, + { + name: "valid label resource type", + field: labelsField, + resourceType: "deployment", + }, + { + // An empty resource type is an absent filter for the clear commands. + // The set commands reject it separately. + name: "empty resource type passes", + field: annotationsField, + resourceType: "", + }, + { + name: "unknown resource type", + field: annotationsField, + resourceType: "deploymnet", + wantErr: true, + }, + { + // Keda resources do not flow user labels through, so they are + // annotation-only resource types. + name: "keda resource type is valid for annotations", + field: annotationsField, + resourceType: "keda_scaled_object", + }, + { + name: "keda resource type is invalid for labels", + field: labelsField, + resourceType: "keda_scaled_object", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateResourceType(test.field, test.resourceType) + if test.wantErr { + if err == nil { + t.Fatalf("validateResourceType(%s, %q) = nil, want error", test.field.Name, test.resourceType) + } + if !strings.Contains(err.Error(), "Invalid resource-type specified, valid resource types include: ") { + t.Errorf("validateResourceType(%s, %q) error = %q, want the valid type list", test.field.Name, test.resourceType, err.Error()) + } + for _, valid := range test.field.ResourceTypes { + if !strings.Contains(err.Error(), valid) { + t.Errorf("validateResourceType(%s, %q) error = %q, missing valid type %s", test.field.Name, test.resourceType, err.Error(), valid) + } + } + return + } + + if err != nil { + t.Errorf("validateResourceType(%s, %q) returned error: %s", test.field.Name, test.resourceType, err) + } + }) + } +} + +func TestMetadataPropertyName(t *testing.T) { + // Annotations are stored unprefixed and labels under a labels. prefix, which is + // what getAnnotation and getLabel already build. + if got := metadataPropertyName(annotationsField, "web", "deployment"); got != "web.deployment" { + t.Errorf("metadataPropertyName(annotations, web, deployment) = %q, want web.deployment", got) + } + if got := metadataPropertyName(labelsField, "web", "deployment"); got != "labels.web.deployment" { + t.Errorf("metadataPropertyName(labels, web, deployment) = %q, want labels.web.deployment", got) + } + if got := metadataPropertyName(annotationsField, GlobalProcessType, "pod"); got != "--global.pod" { + t.Errorf("metadataPropertyName(annotations, --global, pod) = %q, want --global.pod", got) + } +} diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index d88f0177d..c2e9b805d 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -396,101 +396,34 @@ func ReportLabelsSingleApp(appName string, format string, processType string, re // collectAnnotationsEntries scans the property store for annotation entries on appName, // optionally filtered by processType/resourceType. func collectAnnotationsEntries(appName string, processType string, resourceType string) ([]annotationsLabelsReportEntry, error) { - properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, "") - if err != nil { - return nil, fmt.Errorf("Unable to get property list: %w", err) - } - - knownResourceTypes := map[string]bool{} - for _, rt := range AnnotationResourceTypes { - knownResourceTypes[rt] = true - } - - entries := []annotationsLabelsReportEntry{} - for propertyName := range properties { - if isReservedAnnotationProperty(propertyName) { - continue - } - - dot := strings.LastIndex(propertyName, ".") - if dot <= 0 || dot == len(propertyName)-1 { - continue - } - - propProcessType := propertyName[:dot] - propResourceType := propertyName[dot+1:] - if !knownResourceTypes[propResourceType] { - continue - } - - if processType != "" && propProcessType != processType { - continue - } - if resourceType != "" && propResourceType != resourceType { - continue - } - - annotationMap, err := getAnnotation(appName, propProcessType, propResourceType) - if err != nil { - return nil, fmt.Errorf("Unable to read annotation %s: %w", propertyName, err) - } - - for key, value := range annotationMap { - entries = append(entries, annotationsLabelsReportEntry{ - ProcessType: propProcessType, - ResourceType: propResourceType, - Key: key, - Value: value, - }) - } - } - - return entries, nil + return collectMetadataEntries(annotationsField, appName, processType, resourceType, getAnnotation) } // collectLabelsEntries scans the property store for label entries on appName, // optionally filtered by processType/resourceType. func collectLabelsEntries(appName string, processType string, resourceType string) ([]annotationsLabelsReportEntry, error) { - properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, "labels.") - if err != nil { - return nil, fmt.Errorf("Unable to get property list: %w", err) - } + return collectMetadataEntries(labelsField, appName, processType, resourceType, getLabel) +} - knownResourceTypes := map[string]bool{} - for _, rt := range LabelResourceTypes { - knownResourceTypes[rt] = true +// collectMetadataEntries flattens every stored scope a field owns on appName into +// report entries, reading the scopes through the same scan the clear commands use. +func collectMetadataEntries(field metadataField, appName string, processType string, resourceType string, read func(string, string, string) (map[string]string, error)) ([]annotationsLabelsReportEntry, error) { + properties, err := matchingMetadataProperties(field, appName, processType, resourceType) + if err != nil { + return nil, err } entries := []annotationsLabelsReportEntry{} - for propertyName := range properties { - suffix := strings.TrimPrefix(propertyName, "labels.") - dot := strings.LastIndex(suffix, ".") - if dot <= 0 || dot == len(suffix)-1 { - continue - } - - propProcessType := suffix[:dot] - propResourceType := suffix[dot+1:] - if !knownResourceTypes[propResourceType] { - continue - } - - if processType != "" && propProcessType != processType { - continue - } - if resourceType != "" && propResourceType != resourceType { - continue - } - - labelMap, err := getLabel(appName, propProcessType, propResourceType) + for _, property := range properties { + values, err := read(appName, property.ProcessType, property.ResourceType) if err != nil { - return nil, fmt.Errorf("Unable to read label %s: %w", propertyName, err) + return nil, fmt.Errorf("Unable to read %s %s: %w", field.Singular, property.PropertyName, err) } - for key, value := range labelMap { + for key, value := range values { entries = append(entries, annotationsLabelsReportEntry{ - ProcessType: propProcessType, - ResourceType: propResourceType, + ProcessType: property.ProcessType, + ResourceType: property.ResourceType, Key: key, Value: value, }) diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index 8c7d33678..b1af75618 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -17,9 +17,12 @@ Manage scheduler-k3s settings for an app Additional commands:` helpContent = ` + scheduler-k3s:annotations:clear [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Clear all annotations for an app or a single process-type/resource-type scope scheduler-k3s:annotations:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear an annotation for a given app/process-type/resource-type combination + scheduler-k3s:annotations:set --replace [ ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Replace the entire annotation map for a given app/process-type/resource-type combination scheduler-k3s:annotations:report [|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s annotations report for one or more apps scheduler-k3s:autoscaling-auth:set [<--metadata key=value>...], Set or clear a scheduler-k3s autoscaling keda trigger authentication resource for an app + scheduler-k3s:autoscaling-auth:set --replace <--metadata key=value> [<--metadata key=value>...], Replace the entire metadata map for a scheduler-k3s autoscaling keda trigger authentication resource scheduler-k3s:autoscaling-auth:report [|--global] [--format stdout|json] [--include-metadata], Displays a scheduler-k3s autoscaling auth report for one or more apps scheduler-k3s:charts:report [] [--format stdout|json], Displays a scheduler-k3s chart override report scheduler-k3s:charts:set (), Set or clear a chart-specific helm value @@ -28,9 +31,13 @@ Additional commands:` scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster scheduler-k3s:ensure-charts, Ensures the k3s charts are installed scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster + scheduler-k3s:labels:clear [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Clear all labels for an app or a single process-type/resource-type scope scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination + scheduler-k3s:labels:set --replace [ ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Replace the entire label map for a given app/process-type/resource-type combination scheduler-k3s:labels:report [|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps + scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE], Clear all node-level kernel sysctls for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:set () [--global|--profile PROFILE], Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile + scheduler-k3s:node-sysctls:set --replace [ ...] [--global|--profile PROFILE], Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile scheduler-k3s:node-sysctls:report [--format stdout|json], Displays the node-level kernel sysctls applied to each scope scheduler-k3s:preview [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app scheduler-k3s:profiles:add [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 24631400c..5a7e07dde 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -18,22 +18,50 @@ func main() { var err error switch subcommand { + case "annotations:clear": + args := flag.NewFlagSet("scheduler-k3s:annotations:clear", flag.ExitOnError) + global := args.Bool("global", false, "--global: clear global annotations") + processType := args.String("process-type", "", "--process-type: filter by process-type") + resourceType := args.String("resource-type", "", "--resource-type: filter by resource-type") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + if *global { + appName = "--global" + } + + err = scheduler_k3s.CommandAnnotationsClear(scheduler_k3s.MetadataClearInput{ + AppName: appName, + ProcessType: *processType, + ResourceType: *resourceType, + }) case "annotations:set": args := flag.NewFlagSet("scheduler-k3s:annotations:set", flag.ExitOnError) global := args.Bool("global", false, "--global: set a global property") processType := args.String("process-type", "", "--process-type: scope to process-type") resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type") + replace := args.Bool("replace", false, "--replace: replace the entire annotation map for the scope") args.Parse(os.Args[2:]) - appName := args.Arg(0) - property := args.Arg(1) - value := args.Arg(2) - if *global { - appName = "--global" - property = args.Arg(0) - value = args.Arg(1) + input := scheduler_k3s.MetadataSetInput{ + AppName: args.Arg(0), + ProcessType: *processType, + ResourceType: *resourceType, + Replace: *replace, } - err = scheduler_k3s.CommandAnnotationsSet(appName, *processType, *resourceType, property, value) + skip := 1 + if *global { + input.AppName = "--global" + skip = 0 + } + + if *replace { + input.Pairs = common.VarArgs(args.Args(), skip) + } else { + input.Key = args.Arg(skip) + input.Value = args.Arg(skip + 1) + } + + err = scheduler_k3s.CommandAnnotationsSet(input) case "annotations:report": args := flag.NewFlagSet("scheduler-k3s:annotations:report", flag.ExitOnError) global := args.Bool("global", false, "--global: show the global report") @@ -55,6 +83,7 @@ func main() { args := flag.NewFlagSet("scheduler-k3s:autoscaling-auth:set", flag.ExitOnError) global := args.Bool("global", false, "--global: set a global property") metadata := args.StringToString("metadata", map[string]string{}, "--metadata: a key=value map of parameter metadata") + replace := args.Bool("replace", false, "--replace: replace the entire metadata map for the trigger") args.Parse(os.Args[2:]) appName := args.Arg(0) trigger := args.Arg(1) @@ -62,7 +91,7 @@ func main() { appName = "--global" trigger = args.Arg(0) } - err = scheduler_k3s.CommandAutoscalingAuthSet(appName, trigger, *metadata, *global) + err = scheduler_k3s.CommandAutoscalingAuthSet(appName, trigger, *metadata, *global, *replace) case "autoscaling-auth:report": args := flag.NewFlagSet("scheduler-k3s:autoscaling-auth:report", flag.ExitOnError) global := args.Bool("global", false, "--global: show the global report") @@ -131,22 +160,50 @@ func main() { kubeletArgs := args.StringSlice("kubelet-args", []string{}, "kubelet-args: repeatable key=value kubelet arguments (e.g., --kubelet-args key=value)") args.Parse(os.Args[2:]) err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling, *kubeletArgs) + case "labels:clear": + args := flag.NewFlagSet("scheduler-k3s:labels:clear", flag.ExitOnError) + global := args.Bool("global", false, "--global: clear global labels") + processType := args.String("process-type", "", "--process-type: filter by process-type") + resourceType := args.String("resource-type", "", "--resource-type: filter by resource-type") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + if *global { + appName = "--global" + } + + err = scheduler_k3s.CommandLabelsClear(scheduler_k3s.MetadataClearInput{ + AppName: appName, + ProcessType: *processType, + ResourceType: *resourceType, + }) case "labels:set": args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError) global := args.Bool("global", false, "--global: set a global property") processType := args.String("process-type", "", "--process-type: scope to process-type") resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type") + replace := args.Bool("replace", false, "--replace: replace the entire label map for the scope") args.Parse(os.Args[2:]) - appName := args.Arg(0) - property := args.Arg(1) - value := args.Arg(2) - if *global { - appName = "--global" - property = args.Arg(0) - value = args.Arg(1) + input := scheduler_k3s.MetadataSetInput{ + AppName: args.Arg(0), + ProcessType: *processType, + ResourceType: *resourceType, + Replace: *replace, } - err = scheduler_k3s.CommandLabelsSet(appName, *processType, *resourceType, property, value) + skip := 1 + if *global { + input.AppName = "--global" + skip = 0 + } + + if *replace { + input.Pairs = common.VarArgs(args.Args(), skip) + } else { + input.Key = args.Arg(skip) + input.Value = args.Arg(skip + 1) + } + + err = scheduler_k3s.CommandLabelsSet(input) case "labels:report": args := flag.NewFlagSet("scheduler-k3s:labels:report", flag.ExitOnError) global := args.Bool("global", false, "--global: show the global report") @@ -164,18 +221,39 @@ func main() { appName = "--global" } err = scheduler_k3s.CommandLabelsReport(appName, *format, *processType, *resourceType, infoFlag) - case "node-sysctls:set": - args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError) + case "node-sysctls:clear": + args := flag.NewFlagSet("scheduler-k3s:node-sysctls:clear", flag.ExitOnError) global := args.Bool("global", false, "--global: scope to all nodes without a node profile") profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes") args.Parse(os.Args[2:]) - key := args.Arg(0) - value := args.Arg(1) if *global && *profileName != "" { err = fmt.Errorf("Only one of --global and --profile may be specified") break } - err = scheduler_k3s.CommandNodeSysctlsSet(*profileName, key, value) + err = scheduler_k3s.CommandNodeSysctlsClear(*profileName) + case "node-sysctls:set": + args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: scope to all nodes without a node profile") + profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes") + replace := args.Bool("replace", false, "--replace: replace the entire sysctl map for the scope") + args.Parse(os.Args[2:]) + if *global && *profileName != "" { + err = fmt.Errorf("Only one of --global and --profile may be specified") + break + } + + input := scheduler_k3s.NodeSysctlsSetInput{ + ProfileName: *profileName, + Replace: *replace, + } + if *replace { + input.Pairs = args.Args() + } else { + input.Key = args.Arg(0) + input.Value = args.Arg(1) + } + + err = scheduler_k3s.CommandNodeSysctlsSet(input) case "node-sysctls:report": args := flag.NewFlagSet("scheduler-k3s:node-sysctls:report", flag.ExitOnError) format := args.String("format", "stdout", "format: [ stdout | json ]") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 7ca27f354..641bfaa02 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -20,56 +20,70 @@ import ( "helm.sh/helm/v3/pkg/chartutil" ) -// CommandAnnotationsSet set or clear a scheduler-k3s annotation for an app -func CommandAnnotationsSet(appName string, processType string, resourceType string, key string, value string) error { - if resourceType == "" { - return fmt.Errorf("Missing resource-type") - } - - if processType == "" { - processType = GlobalProcessType - } - - property := fmt.Sprintf("%s.%s", processType, resourceType) - if value == "" { - if err := common.PropertyMapDelete("scheduler-k3s", appName, property, key); err != nil { - return fmt.Errorf("Unable to delete property map entry: %w", err) - } - return nil - } - - if err := common.PropertyMapSet("scheduler-k3s", appName, property, key, value); err != nil { - return fmt.Errorf("Unable to set property map entry: %w", err) - } - - return nil +// CommandAnnotationsSet sets, clears or replaces scheduler-k3s annotations for an app +func CommandAnnotationsSet(input MetadataSetInput) error { + return commandMetadataSet(annotationsField, input) } -// CommandNodeSysctlsSet sets or clears a node-level kernel sysctl for a scope -func CommandNodeSysctlsSet(profileName string, key string, value string) error { - if key == "" { +// CommandAnnotationsClear clears scheduler-k3s annotations for an app +func CommandAnnotationsClear(input MetadataClearInput) error { + return commandMetadataClear(annotationsField, input) +} + +// NodeSysctlsSetInput captures the inputs accepted by node-sysctls:set. +// Key/Value and Pairs are mutually exclusive: Pairs is populated only under +// --replace, where the positional arguments are key=value pairs rather than a +// single sysctl name and value. +type NodeSysctlsSetInput struct { + ProfileName string + Key string + Value string + Pairs []string + Replace bool +} + +// CommandNodeSysctlsSet sets, clears or replaces node-level kernel sysctls for a scope +func CommandNodeSysctlsSet(input NodeSysctlsSetInput) error { + if err := verifyNodeSysctlsScope(input.ProfileName); err != nil { + return err + } + + property := getNodeSysctlsProperty(input.ProfileName) + if input.Replace { + if len(input.Pairs) == 0 { + return fmt.Errorf("Must specify at least one key=value pair, use scheduler-k3s:node-sysctls:clear to remove all sysctls") + } + + parsed, err := parseMetadataPairs(input.Pairs) + if err != nil { + return err + } + + removed, err := removedSysctlNames(property, parsed) + if err != nil { + return err + } + + if err := common.PropertyMapWrite("scheduler-k3s", "--global", property, parsed); err != nil { + return fmt.Errorf("Unable to write property map: %w", err) + } + + warnUnmanagedSysctls(removed) + return CreateOrUpdateNodeSysctls(context.Background()) + } + + if input.Key == "" { return fmt.Errorf("Missing sysctl name") } - if profileName != "" { - if err := verifyNodeProfileExists(profileName); err != nil { - return err - } - - if err := verifyNodeProfileSysctlsSupported(profileName); err != nil { - return err - } - } - - property := getNodeSysctlsProperty(profileName) - if value == "" { - if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, key); err != nil { + if input.Value == "" { + if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, input.Key); err != nil { return fmt.Errorf("Unable to delete property map entry: %w", err) } - common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", key)) + warnUnmanagedSysctls([]string{input.Key}) } else { - if err := common.PropertyMapSet("scheduler-k3s", "--global", property, key, value); err != nil { + if err := common.PropertyMapSet("scheduler-k3s", "--global", property, input.Key, input.Value); err != nil { return fmt.Errorf("Unable to set property map entry: %w", err) } } @@ -77,6 +91,68 @@ func CommandNodeSysctlsSet(profileName string, key string, value string) error { return CreateOrUpdateNodeSysctls(context.Background()) } +// CommandNodeSysctlsClear clears every node-level kernel sysctl for a scope +func CommandNodeSysctlsClear(profileName string) error { + if err := verifyNodeSysctlsScope(profileName); err != nil { + return err + } + + property := getNodeSysctlsProperty(profileName) + removed, err := removedSysctlNames(property, map[string]string{}) + if err != nil { + return err + } + + if err := common.PropertyDelete("scheduler-k3s", "--global", property); err != nil { + return fmt.Errorf("Unable to delete property: %w", err) + } + + warnUnmanagedSysctls(removed) + return CreateOrUpdateNodeSysctls(context.Background()) +} + +// verifyNodeSysctlsScope returns an error when a profile-scoped command names a profile +// that does not exist or cannot back a helm release. +func verifyNodeSysctlsScope(profileName string) error { + if profileName == "" { + return nil + } + + if err := verifyNodeProfileExists(profileName); err != nil { + return err + } + + return verifyNodeProfileSysctlsSupported(profileName) +} + +// removedSysctlNames returns the sysctls a scope currently manages that the next +// stored map no longer will, so they can be warned about before the write happens. +func removedSysctlNames(property string, next map[string]string) ([]string, error) { + current, err := common.PropertyMapGet("scheduler-k3s", "--global", property) + if err != nil { + return nil, fmt.Errorf("Unable to get property map: %w", err) + } + + removed := []string{} + for key := range current { + if _, ok := next[key]; !ok { + removed = append(removed, key) + } + } + + sort.Strings(removed) + return removed, nil +} + +// warnUnmanagedSysctls notes that dokku has stopped managing a sysctl. The last +// value written stays on affected nodes until they reboot, which is how sysctl -w +// behaves everywhere else. +func warnUnmanagedSysctls(names []string) { + for _, name := range names { + common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", name)) + } +} + // CommandNodeSysctlsReport displays the configured node-level kernel sysctls func CommandNodeSysctlsReport(format string) error { if format != "stdout" && format != "json" { @@ -149,8 +225,8 @@ func verifyNodeProfileSysctlsSupported(profileName string) error { return nil } -// CommandAutoscalingAuthSet set or clear a scheduler-k3s autoscaling keda trigger authentication object for an app -func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool) error { +// CommandAutoscalingAuthSet sets, clears or replaces a scheduler-k3s autoscaling keda trigger authentication object for an app +func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool, replace bool) error { if global { appName = "--global" } @@ -165,18 +241,17 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri return fmt.Errorf("Missing trigger type argument") } + if replace && len(metadata) == 0 { + return fmt.Errorf("Must specify at least one --metadata flag, omit --replace to remove all metadata for the trigger") + } + + if len(metadata) == 0 || replace { + if err := deleteTriggerAuthProperties(appName, trigger); err != nil { + return err + } + } + if len(metadata) == 0 { - properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, fmt.Sprintf("%s%s.", TriggerAuthPropertyPrefix, trigger)) - if err != nil { - return fmt.Errorf("Unable to get property list: %w", err) - } - - for key := range properties { - if err := common.PropertyDelete("scheduler-k3s", appName, key); err != nil { - return fmt.Errorf("Unable to delete property: %w", err) - } - } - if appName == "--global" { helmAgent, err := NewHelmAgent("keda", DeployLogPrinter) if err != nil { @@ -196,11 +271,11 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri if err := common.PropertyWrite("scheduler-k3s", appName, fmt.Sprintf("%s%s.%s", TriggerAuthPropertyPrefix, trigger, key), value); err != nil { return fmt.Errorf("Unable to set property: %w", err) } - - common.LogInfo1("Trigger authentication settings saved") - common.LogVerbose("Resources will be created or updated on next deploy") } + common.LogInfo1("Trigger authentication settings saved") + common.LogVerbose("Resources will be created or updated on next deploy") + if appName == "--global" { err := applyKedaClusterTriggerAuthentications(context.Background(), trigger, metadata) if err != nil { @@ -211,6 +286,22 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri return nil } +// deleteTriggerAuthProperties removes every stored metadata key for one trigger. +func deleteTriggerAuthProperties(appName string, trigger string) error { + properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, fmt.Sprintf("%s%s.", TriggerAuthPropertyPrefix, trigger)) + if err != nil { + return fmt.Errorf("Unable to get property list: %w", err) + } + + for key := range properties { + if err := common.PropertyDelete("scheduler-k3s", appName, key); err != nil { + return fmt.Errorf("Unable to delete property: %w", err) + } + } + + return nil +} + // CommandAutoscalingAuthReport displays a scheduler-k3s autoscaling keda trigger authentication report for one or more apps func CommandAutoscalingAuthReport(appName string, format string, includeMetadata bool, infoFlag string) error { if len(appName) == 0 { @@ -1237,29 +1328,14 @@ func CommandEnsureCharts(forceInstall bool, forceChartNames []string) error { return nil } -// CommandLabelsSet set or clear a scheduler-k3s label for an app -func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error { - if resourceType == "" { - return fmt.Errorf("Missing resource-type") - } +// CommandLabelsSet sets, clears or replaces scheduler-k3s labels for an app +func CommandLabelsSet(input MetadataSetInput) error { + return commandMetadataSet(labelsField, input) +} - if processType == "" { - processType = GlobalProcessType - } - - property := fmt.Sprintf("labels.%s.%s", processType, resourceType) - if value == "" { - if err := common.PropertyMapDelete("scheduler-k3s", appName, property, key); err != nil { - return fmt.Errorf("Unable to delete property map entry: %w", err) - } - return nil - } - - if err := common.PropertyMapSet("scheduler-k3s", appName, property, key, value); err != nil { - return fmt.Errorf("Unable to set property map entry: %w", err) - } - - return nil +// CommandLabelsClear clears scheduler-k3s labels for an app +func CommandLabelsClear(input MetadataClearInput) error { + return commandMetadataClear(labelsField, input) } // CommandProfilesAdd adds a node profile to the k3s cluster diff --git a/tests/unit/scheduler-k3s-annotations.bats b/tests/unit/scheduler-k3s-annotations.bats index 8ff04b1a3..78c840d4c 100644 --- a/tests/unit/scheduler-k3s-annotations.bats +++ b/tests/unit/scheduler-k3s-annotations.bats @@ -10,10 +10,8 @@ setup() { teardown() { dokku --force apps:destroy $TEST_APP >/dev/null 2>/dev/null || true dokku --force apps:destroy ${TEST_APP}-2 >/dev/null 2>/dev/null || true - for resource in deployment service pod cronjob job secret ingress serviceaccount; do - dokku scheduler-k3s:annotations:set --global --resource-type "$resource" foo "" >/dev/null 2>/dev/null || true - dokku scheduler-k3s:annotations:set --global --resource-type "$resource" bar "" >/dev/null 2>/dev/null || true - done + dokku scheduler-k3s:annotations:clear --global >/dev/null 2>/dev/null || true + dokku scheduler-k3s:node-sysctls:clear --global >/dev/null 2>/dev/null || true global_teardown } @@ -113,3 +111,166 @@ teardown() { assert_failure assert_output_contains "Invalid flag passed" } + +@test "(scheduler-k3s:annotations:set) --replace writes the declared map in one call" { + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment keep old-value" + assert_success + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment drop dropped-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment keep=new-value add=added-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.add,global.deployment.keep" + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r '.\"global.deployment.keep\"'" + assert_success + assert_output "new-value" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment --process-type web scoped=scoped-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.add,global.deployment.keep,web.deployment.scoped" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace --global --resource-type deployment global-key=global-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report --global --format json | jq -r '.\"global.deployment.global-key\"'" + assert_success + assert_output "global-value" +} + +@test "(scheduler-k3s:annotations:set) --replace stores empty values and rejects malformed pair lists" { + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment empty=" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r '.\"global.deployment.empty\"'" + assert_success + assert_output "" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment" + assert_failure + assert_output_contains "Must specify at least one key=value pair, use scheduler-k3s:annotations:clear to remove all annotations" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment nodelimiter" + assert_failure + assert_output_contains "Invalid key=value pair: nodelimiter" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deployment foo=bar foo=qux" + assert_failure + assert_output_contains "Duplicate key specified: foo" + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.empty" +} + +@test "(scheduler-k3s:annotations:set) rejects an unknown resource-type and a missing app" { + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deploymnet foo bar" + assert_failure + assert_output_contains "Invalid resource-type specified, valid resource types include:" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace $TEST_APP --resource-type deploymnet foo=bar" + assert_failure + assert_output_contains "Invalid resource-type specified, valid resource types include:" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set ${TEST_APP}-missing --resource-type deployment foo bar" + assert_failure + assert_output_contains "App ${TEST_APP}-missing does not exist" + + run /bin/bash -c "dokku scheduler-k3s:annotations:set --replace ${TEST_APP}-missing --resource-type deployment foo=bar" + assert_failure + assert_output_contains "App ${TEST_APP}-missing does not exist" +} + +@test "(scheduler-k3s:annotations:clear) removes annotations, honouring filters" { + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment foo bar" + assert_success + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --process-type web --resource-type deployment baz qux" + assert_success + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type service alpha beta" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP --process-type web" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.foo,global.service.alpha" + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP --resource-type service" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.foo" + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json" + assert_success + assert_output "{}" +} + +@test "(scheduler-k3s:annotations:clear) leaves neighbouring scheduler-k3s properties alone" { + run /bin/bash -c "dokku scheduler-k3s:set $TEST_APP deploy-timeout 300s" + assert_success + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144" + assert_success + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-1" + assert_success + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment foo bar" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report $TEST_APP --format json" + assert_success + assert_output "{}" + + run /bin/bash -c "dokku --quiet scheduler-k3s:report $TEST_APP --scheduler-k3s-deploy-timeout" + assert_success + assert_output "300s" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\".\"vm.max_map_count\"'" + assert_success + assert_output "262144" + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r '.\"datadog.apiKey\"'" + assert_success + assert_output "secret-1" +} + +@test "(scheduler-k3s:annotations:clear) supports --global and requires an app" { + run /bin/bash -c "dokku scheduler-k3s:annotations:set --global --resource-type deployment foo global-val" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear" + assert_failure + assert_output_contains "Please specify an app to run the command on" + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear ${TEST_APP}-missing" + assert_failure + assert_output_contains "App ${TEST_APP}-missing does not exist" + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear $TEST_APP --resource-type deploymnet" + assert_failure + assert_output_contains "Invalid resource-type specified, valid resource types include:" + + run /bin/bash -c "dokku scheduler-k3s:annotations:report --global --format json | jq -r '.\"global.deployment.foo\"'" + assert_success + assert_output "global-val" + + run /bin/bash -c "dokku scheduler-k3s:annotations:clear --global" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:report --global --format json" + assert_success + assert_output "{}" +} diff --git a/tests/unit/scheduler-k3s-autoscaling-auth.bats b/tests/unit/scheduler-k3s-autoscaling-auth.bats index 3b3428962..6f8bf0255 100644 --- a/tests/unit/scheduler-k3s-autoscaling-auth.bats +++ b/tests/unit/scheduler-k3s-autoscaling-auth.bats @@ -149,3 +149,44 @@ teardown() { assert_failure assert_output_contains "Invalid flag passed" } + +@test "(scheduler-k3s:autoscaling-auth:set) --replace drops metadata keys not named" { + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-1 --metadata appKey=secret-2 --metadata datadogSite=us5.datadoghq.com" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-3" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "datadog.apiKey,datadog.appKey,datadog.datadogSite" + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set --replace $TEST_APP datadog --metadata apiKey=secret-4" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "datadog.apiKey" + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r '.\"datadog.apiKey\"'" + assert_success + assert_output "secret-4" +} + +@test "(scheduler-k3s:autoscaling-auth:set) --replace requires metadata and leaves other triggers alone" { + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP datadog --metadata apiKey=secret-1" + assert_success + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set $TEST_APP memory --metadata some-key=some-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set --replace $TEST_APP datadog" + assert_failure + assert_output_contains "Must specify at least one --metadata flag, omit --replace to remove all metadata for the trigger" + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:set --replace $TEST_APP datadog --metadata appKey=secret-2" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:autoscaling-auth:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "datadog.appKey,memory.some-key" +} diff --git a/tests/unit/scheduler-k3s-labels.bats b/tests/unit/scheduler-k3s-labels.bats index ae007b45c..165540d4d 100644 --- a/tests/unit/scheduler-k3s-labels.bats +++ b/tests/unit/scheduler-k3s-labels.bats @@ -10,10 +10,7 @@ setup() { teardown() { dokku --force apps:destroy $TEST_APP >/dev/null 2>/dev/null || true dokku --force apps:destroy ${TEST_APP}-2 >/dev/null 2>/dev/null || true - for resource in deployment service pod cronjob job secret ingress serviceaccount; do - dokku scheduler-k3s:labels:set --global --resource-type "$resource" foo "" >/dev/null 2>/dev/null || true - dokku scheduler-k3s:labels:set --global --resource-type "$resource" bar "" >/dev/null 2>/dev/null || true - done + dokku scheduler-k3s:labels:clear --global >/dev/null 2>/dev/null || true global_teardown } @@ -113,3 +110,136 @@ teardown() { assert_failure assert_output_contains "Invalid flag passed" } + +@test "(scheduler-k3s:labels:set) --replace writes the declared map in one call" { + run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deployment keep old-value" + assert_success + run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deployment drop dropped-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment keep=new-value add=added-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.add,global.deployment.keep" + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r '.\"global.deployment.keep\"'" + assert_success + assert_output "new-value" + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment --process-type web scoped=scoped-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.add,global.deployment.keep,web.deployment.scoped" + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace --global --resource-type deployment global-key=global-value" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report --global --format json | jq -r '.\"global.deployment.global-key\"'" + assert_success + assert_output "global-value" +} + +@test "(scheduler-k3s:labels:set) --replace stores empty values and rejects malformed pair lists" { + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment empty=" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r '.\"global.deployment.empty\"'" + assert_success + assert_output "" + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment" + assert_failure + assert_output_contains "Must specify at least one key=value pair, use scheduler-k3s:labels:clear to remove all labels" + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment nodelimiter" + assert_failure + assert_output_contains "Invalid key=value pair: nodelimiter" + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deployment foo=bar foo=qux" + assert_failure + assert_output_contains "Duplicate key specified: foo" + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.empty" +} + +@test "(scheduler-k3s:labels:set) rejects an unknown resource-type and a missing app" { + run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deploymnet foo bar" + assert_failure + assert_output_contains "Invalid resource-type specified, valid resource types include:" + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace $TEST_APP --resource-type deploymnet foo=bar" + assert_failure + assert_output_contains "Invalid resource-type specified, valid resource types include:" + + run /bin/bash -c "dokku scheduler-k3s:labels:set ${TEST_APP}-missing --resource-type deployment foo bar" + assert_failure + assert_output_contains "App ${TEST_APP}-missing does not exist" + + run /bin/bash -c "dokku scheduler-k3s:labels:set --replace ${TEST_APP}-missing --resource-type deployment foo=bar" + assert_failure + assert_output_contains "App ${TEST_APP}-missing does not exist" +} + +@test "(scheduler-k3s:labels:clear) removes labels, honouring filters" { + run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type deployment foo bar" + assert_success + run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --process-type web --resource-type deployment baz qux" + assert_success + run /bin/bash -c "dokku scheduler-k3s:labels:set $TEST_APP --resource-type service alpha beta" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP --process-type web" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.foo,global.service.alpha" + + run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP --resource-type service" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json | jq -r 'keys | sort | join(\",\")'" + assert_success + assert_output "global.deployment.foo" + + run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report $TEST_APP --format json" + assert_success + assert_output "{}" +} + +@test "(scheduler-k3s:labels:clear) supports --global and requires an app" { + run /bin/bash -c "dokku scheduler-k3s:labels:set --global --resource-type deployment foo global-val" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:clear" + assert_failure + assert_output_contains "Please specify an app to run the command on" + + run /bin/bash -c "dokku scheduler-k3s:labels:clear ${TEST_APP}-missing" + assert_failure + assert_output_contains "App ${TEST_APP}-missing does not exist" + + run /bin/bash -c "dokku scheduler-k3s:labels:clear $TEST_APP --resource-type deploymnet" + assert_failure + assert_output_contains "Invalid resource-type specified, valid resource types include:" + + run /bin/bash -c "dokku scheduler-k3s:labels:report --global --format json | jq -r '.\"global.deployment.foo\"'" + assert_success + assert_output "global-val" + + run /bin/bash -c "dokku scheduler-k3s:labels:clear --global" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:labels:report --global --format json" + assert_success + assert_output "{}" +} diff --git a/tests/unit/scheduler-k3s-node-sysctls.bats b/tests/unit/scheduler-k3s-node-sysctls.bats new file mode 100644 index 000000000..9cd5f154d --- /dev/null +++ b/tests/unit/scheduler-k3s-node-sysctls.bats @@ -0,0 +1,87 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + global_setup +} + +teardown() { + dokku scheduler-k3s:node-sysctls:clear --global >/dev/null 2>/dev/null || true + dokku scheduler-k3s:node-sysctls:clear --profile edge-workers >/dev/null 2>/dev/null || true + dokku scheduler-k3s:profiles:remove edge-workers >/dev/null 2>/dev/null || true + global_teardown +} + +@test "(scheduler-k3s:node-sysctls:set) --replace writes the declared map in one call" { + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144" + assert_success + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.swappiness 10" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.swappiness=20 vm.overcommit_memory=1" + assert_success + assert_output_contains "Removing vm.max_map_count stops dokku managing it" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\" | keys | sort | join(\",\")'" + assert_success + assert_output "vm.overcommit_memory,vm.swappiness" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\".\"vm.swappiness\"'" + assert_success + assert_output "20" + + run /bin/bash -c "dokku scheduler-k3s:profiles:add edge-workers --role worker" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --profile edge-workers vm.swappiness=60" + assert_success + + # a profile scope inherits the global sysctls and overrides them on conflict + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"edge-workers\" | to_entries | sort_by(.key) | map(\"\(.key)=\(.value)\") | join(\",\")'" + assert_success + assert_output "vm.overcommit_memory=1,vm.swappiness=60" +} + +@test "(scheduler-k3s:node-sysctls:set) --replace rejects malformed pair lists without writing" { + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global" + assert_failure + assert_output_contains "Must specify at least one key=value pair, use scheduler-k3s:node-sysctls:clear to remove all sysctls" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global nodelimiter" + assert_failure + assert_output_contains "Invalid key=value pair: nodelimiter" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.swappiness=10 vm.swappiness=20" + assert_failure + assert_output_contains "Duplicate key specified: vm.swappiness" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\" | keys | join(\",\")'" + assert_success + assert_output "vm.max_map_count" +} + +@test "(scheduler-k3s:node-sysctls:clear) removes every sysctl for a scope" { + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --replace --global vm.max_map_count=262144 vm.swappiness=10" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --global --profile edge-workers" + assert_failure + assert_output_contains "Only one of --global and --profile may be specified" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --profile missing-profile" + assert_failure + assert_output_contains "Node profile missing-profile not found" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:clear --global" + assert_success + assert_output_contains "Removing vm.max_map_count stops dokku managing it" + assert_output_contains "Removing vm.swappiness stops dokku managing it" + + run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\" | length'" + assert_success + assert_output "0" +}