feat: add a whole-set form for scheduler-k3s metadata

The `scheduler-k3s` metadata commands were all per-key, so matching an app's stored annotations, labels or node sysctls to a declared set meant reading the report, computing the difference in both directions and issuing one command per added, changed and removed key, with a failure partway through leaving a mixture of the two sets. A `--replace` flag on `annotations:set`, `labels:set`, `node-sysctls:set` and `autoscaling-auth:set` now writes the declared map in a single call, taking trailing `key=value` pairs everywhere except `autoscaling-auth:set`, which keeps its repeatable `--metadata` flag. Every pair is parsed before anything is written, so a rejected pair leaves the stored map untouched, and a pair ending in `=` stores an empty value, which the per-key form cannot express. An empty pair list is rejected rather than treated as a request to remove everything, so a generated list that expands to nothing cannot silently drop an app's metadata; `annotations:clear`, `labels:clear` and `node-sysctls:clear` handle that case, taking the same optional filters as their report commands. The `--resource-type` flag and the app name are now validated on the set commands as well, where a typo previously exited zero and wrote a property that no report and no deploy ever read.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-09-18 05:14:51 -04:00
1 parent 55ffe6809b
commit c37a71b90b
12 files changed
+1271 -195

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
SUBCOMMANDS = subcommands/annotations:clear subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:clear subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:clear subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/scheduler-uses-host-cron triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec
BUILD = commands subcommands triggers
PLUGIN_NAME = scheduler-k3s
+275
View File
@@ -0,0 +1,275 @@
package scheduler_k3s
import (
"fmt"
"sort"
"strings"
"github.com/dokku/dokku/plugins/common"
)
// metadataField selects which of the scheduler-k3s metadata maps a command
// operates on, so the annotations and labels commands can share one
// implementation the way the storage plugin does.
type metadataField struct {
// Name is the plural noun used in user-facing messages.
Name string
// Singular is the singular noun used in user-facing messages.
Singular string
// ClearCommand is the command an empty --replace list points the user at.
ClearCommand string
// PropertyPrefix namespaces the property names this field owns. Annotations
// are stored unprefixed, which is why annotations alone must skip the
// property names reserved by other scheduler-k3s features.
PropertyPrefix string
// SkipReserved drops property names owned by other scheduler-k3s features.
SkipReserved bool
// ResourceTypes lists the kubernetes resource types this field supports.
ResourceTypes []string
}
var annotationsField = metadataField{
Name: "annotations",
Singular: "annotation",
ClearCommand: "scheduler-k3s:annotations:clear",
SkipReserved: true,
ResourceTypes: AnnotationResourceTypes,
}
var labelsField = metadataField{
Name: "labels",
Singular: "label",
ClearCommand: "scheduler-k3s:labels:clear",
PropertyPrefix: "labels.",
ResourceTypes: LabelResourceTypes,
}
// MetadataSetInput captures the inputs accepted by annotations:set and labels:set.
// Key/Value and Pairs are mutually exclusive: Pairs is populated only under
// --replace, where the positional arguments are key=value pairs rather than a
// single key and value.
type MetadataSetInput struct {
AppName string
ProcessType string
ResourceType string
Key string
Value string
Pairs []string
Replace bool
}
// MetadataClearInput captures the inputs accepted by annotations:clear and labels:clear.
// An empty ProcessType or ResourceType filters nothing, matching the report commands.
type MetadataClearInput struct {
AppName string
ProcessType string
ResourceType string
}
// metadataPropertyName returns the property a field stores one scope's map under.
func metadataPropertyName(field metadataField, processType string, resourceType string) string {
return fmt.Sprintf("%s%s.%s", field.PropertyPrefix, processType, resourceType)
}
// verifyMetadataAppName applies the standard app name check, exempting the global scope.
func verifyMetadataAppName(appName string) error {
if appName == "--global" {
return nil
}
return common.VerifyAppName(appName)
}
// validateResourceType reports whether a resource type is one the field supports.
// An empty resource type passes, as the clear commands treat it as an absent filter;
// callers that require one check for it separately.
func validateResourceType(field metadataField, resourceType string) error {
if resourceType == "" {
return nil
}
for _, valid := range field.ResourceTypes {
if valid == resourceType {
return nil
}
}
valid := make([]string, len(field.ResourceTypes))
copy(valid, field.ResourceTypes)
sort.Strings(valid)
return fmt.Errorf("Invalid resource-type specified, valid resource types include: %s", strings.Join(valid, ", "))
}
// parseMetadataPairs turns key=value arguments into a map. A pair without a '='
// or with an empty key is rejected, as is a key specified more than once, since
// a repeated key leaves the declared map ambiguous. A pair ending in '=' stores
// an empty value, which the per-key form cannot express.
func parseMetadataPairs(pairs []string) (map[string]string, error) {
parsed := map[string]string{}
for _, pair := range pairs {
parts := strings.SplitN(pair, "=", 2)
if len(parts) != 2 || parts[0] == "" {
return nil, fmt.Errorf("Invalid key=value pair: %s", pair)
}
if _, ok := parsed[parts[0]]; ok {
return nil, fmt.Errorf("Duplicate key specified: %s", parts[0])
}
parsed[parts[0]] = parts[1]
}
return parsed, nil
}
// setMetadataKey writes a single key on one of the metadata maps. An empty value
// deletes just that key, leaving its siblings in place.
func setMetadataKey(field metadataField, input MetadataSetInput) error {
property := metadataPropertyName(field, input.ProcessType, input.ResourceType)
if input.Value == "" {
if err := common.PropertyMapDelete("scheduler-k3s", input.AppName, property, input.Key); err != nil {
return fmt.Errorf("Unable to delete property map entry: %w", err)
}
return nil
}
if err := common.PropertyMapSet("scheduler-k3s", input.AppName, property, input.Key, input.Value); err != nil {
return fmt.Errorf("Unable to set property map entry: %w", err)
}
return nil
}
// replaceMetadata swaps one scope's whole map for the declared one in a single write.
// Every pair is parsed before anything is written, so a rejected pair leaves the
// stored map untouched.
func replaceMetadata(field metadataField, input MetadataSetInput) error {
if len(input.Pairs) == 0 {
return fmt.Errorf("Must specify at least one key=value pair, use %s to remove all %s", field.ClearCommand, field.Name)
}
parsed, err := parseMetadataPairs(input.Pairs)
if err != nil {
return err
}
property := metadataPropertyName(field, input.ProcessType, input.ResourceType)
if err := common.PropertyMapWrite("scheduler-k3s", input.AppName, property, parsed); err != nil {
return fmt.Errorf("Unable to write property map: %w", err)
}
return nil
}
// clearMetadata removes every metadata property matching the given filters.
func clearMetadata(field metadataField, input MetadataClearInput) error {
properties, err := matchingMetadataProperties(field, input.AppName, input.ProcessType, input.ResourceType)
if err != nil {
return err
}
for _, property := range properties {
if err := common.PropertyDelete("scheduler-k3s", input.AppName, property.PropertyName); err != nil {
return fmt.Errorf("Unable to delete property: %w", err)
}
}
return nil
}
// metadataProperty is one stored metadata scope, as found by scanning the property store.
type metadataProperty struct {
// PropertyName is the name the scope is stored under, prefix included.
PropertyName string
ProcessType string
ResourceType string
}
// matchingMetadataProperties scans the property store for the scopes a field owns on
// appName, optionally filtered by processType/resourceType. Both the report commands
// and the clear commands read through this, so they cannot disagree about what is in scope.
func matchingMetadataProperties(field metadataField, appName string, processType string, resourceType string) ([]metadataProperty, error) {
properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, field.PropertyPrefix)
if err != nil {
return nil, fmt.Errorf("Unable to get property list: %w", err)
}
knownResourceTypes := map[string]bool{}
for _, rt := range field.ResourceTypes {
knownResourceTypes[rt] = true
}
matching := []metadataProperty{}
for propertyName := range properties {
if field.SkipReserved && isReservedAnnotationProperty(propertyName) {
continue
}
suffix := strings.TrimPrefix(propertyName, field.PropertyPrefix)
dot := strings.LastIndex(suffix, ".")
if dot <= 0 || dot == len(suffix)-1 {
continue
}
propProcessType := suffix[:dot]
propResourceType := suffix[dot+1:]
if !knownResourceTypes[propResourceType] {
continue
}
if processType != "" && propProcessType != processType {
continue
}
if resourceType != "" && propResourceType != resourceType {
continue
}
matching = append(matching, metadataProperty{
PropertyName: propertyName,
ProcessType: propProcessType,
ResourceType: propResourceType,
})
}
return matching, nil
}
// commandMetadataSet implements the annotations:set and labels:set commands.
func commandMetadataSet(field metadataField, input MetadataSetInput) error {
if err := verifyMetadataAppName(input.AppName); err != nil {
return err
}
if input.ResourceType == "" {
return fmt.Errorf("Missing resource-type")
}
if err := validateResourceType(field, input.ResourceType); err != nil {
return err
}
if input.ProcessType == "" {
input.ProcessType = GlobalProcessType
}
if input.Replace {
return replaceMetadata(field, input)
}
return setMetadataKey(field, input)
}
// commandMetadataClear implements the annotations:clear and labels:clear commands.
// Unlike the set commands, an omitted process-type or resource-type is an absent
// filter rather than a default scope, matching the report commands.
func commandMetadataClear(field metadataField, input MetadataClearInput) error {
if err := verifyMetadataAppName(input.AppName); err != nil {
return err
}
if err := validateResourceType(field, input.ResourceType); err != nil {
return err
}
return clearMetadata(field, input)
}
+188
View File
@@ -0,0 +1,188 @@
package scheduler_k3s
import (
"reflect"
"strings"
"testing"
)
func TestParseMetadataPairs(t *testing.T) {
tests := []struct {
name string
pairs []string
want map[string]string
}{
{
name: "empty list",
pairs: []string{},
want: map[string]string{},
},
{
name: "multiple pairs",
pairs: []string{"foo=bar", "baz=qux"},
want: map[string]string{"foo": "bar", "baz": "qux"},
},
{
name: "trailing equals stores an empty value",
pairs: []string{"foo="},
want: map[string]string{"foo": ""},
},
{
name: "value containing equals is kept whole",
pairs: []string{"foo=a=b"},
want: map[string]string{"foo": "a=b"},
},
{
name: "kubernetes style key",
pairs: []string{"prometheus.io/scrape=true"},
want: map[string]string{"prometheus.io/scrape": "true"},
},
{
name: "multi-line value",
pairs: []string{"foo=line one\nline two"},
want: map[string]string{"foo": "line one\nline two"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := parseMetadataPairs(test.pairs)
if err != nil {
t.Fatalf("parseMetadataPairs(%q) returned error: %s", test.pairs, err)
}
if !reflect.DeepEqual(got, test.want) {
t.Errorf("parseMetadataPairs(%q) = %v, want %v", test.pairs, got, test.want)
}
})
}
}
func TestParseMetadataPairsErrors(t *testing.T) {
tests := []struct {
name string
pairs []string
want string
}{
{
name: "missing equals",
pairs: []string{"foo"},
want: "Invalid key=value pair: foo",
},
{
name: "empty key",
pairs: []string{"=bar"},
want: "Invalid key=value pair: =bar",
},
{
name: "empty pair",
pairs: []string{""},
want: "Invalid key=value pair: ",
},
{
name: "duplicate key",
pairs: []string{"foo=bar", "foo=qux"},
want: "Duplicate key specified: foo",
},
{
name: "duplicate key with identical value",
pairs: []string{"foo=bar", "foo=bar"},
want: "Duplicate key specified: foo",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := parseMetadataPairs(test.pairs)
if err == nil {
t.Fatalf("parseMetadataPairs(%q) = %v, want error", test.pairs, got)
}
if err.Error() != test.want {
t.Errorf("parseMetadataPairs(%q) error = %q, want %q", test.pairs, err.Error(), test.want)
}
})
}
}
func TestValidateResourceType(t *testing.T) {
tests := []struct {
name string
field metadataField
resourceType string
wantErr bool
}{
{
name: "valid annotation resource type",
field: annotationsField,
resourceType: "deployment",
},
{
name: "valid label resource type",
field: labelsField,
resourceType: "deployment",
},
{
// An empty resource type is an absent filter for the clear commands.
// The set commands reject it separately.
name: "empty resource type passes",
field: annotationsField,
resourceType: "",
},
{
name: "unknown resource type",
field: annotationsField,
resourceType: "deploymnet",
wantErr: true,
},
{
// Keda resources do not flow user labels through, so they are
// annotation-only resource types.
name: "keda resource type is valid for annotations",
field: annotationsField,
resourceType: "keda_scaled_object",
},
{
name: "keda resource type is invalid for labels",
field: labelsField,
resourceType: "keda_scaled_object",
wantErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := validateResourceType(test.field, test.resourceType)
if test.wantErr {
if err == nil {
t.Fatalf("validateResourceType(%s, %q) = nil, want error", test.field.Name, test.resourceType)
}
if !strings.Contains(err.Error(), "Invalid resource-type specified, valid resource types include: ") {
t.Errorf("validateResourceType(%s, %q) error = %q, want the valid type list", test.field.Name, test.resourceType, err.Error())
}
for _, valid := range test.field.ResourceTypes {
if !strings.Contains(err.Error(), valid) {
t.Errorf("validateResourceType(%s, %q) error = %q, missing valid type %s", test.field.Name, test.resourceType, err.Error(), valid)
}
}
return
}
if err != nil {
t.Errorf("validateResourceType(%s, %q) returned error: %s", test.field.Name, test.resourceType, err)
}
})
}
}
func TestMetadataPropertyName(t *testing.T) {
// Annotations are stored unprefixed and labels under a labels. prefix, which is
// what getAnnotation and getLabel already build.
if got := metadataPropertyName(annotationsField, "web", "deployment"); got != "web.deployment" {
t.Errorf("metadataPropertyName(annotations, web, deployment) = %q, want web.deployment", got)
}
if got := metadataPropertyName(labelsField, "web", "deployment"); got != "labels.web.deployment" {
t.Errorf("metadataPropertyName(labels, web, deployment) = %q, want labels.web.deployment", got)
}
if got := metadataPropertyName(annotationsField, GlobalProcessType, "pod"); got != "--global.pod" {
t.Errorf("metadataPropertyName(annotations, --global, pod) = %q, want --global.pod", got)
}
}
+15 -82
View File
@@ -396,101 +396,34 @@ func ReportLabelsSingleApp(appName string, format string, processType string, re
// collectAnnotationsEntries scans the property store for annotation entries on appName,
// optionally filtered by processType/resourceType.
func collectAnnotationsEntries(appName string, processType string, resourceType string) ([]annotationsLabelsReportEntry, error) {
properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, "")
if err != nil {
return nil, fmt.Errorf("Unable to get property list: %w", err)
}
knownResourceTypes := map[string]bool{}
for _, rt := range AnnotationResourceTypes {
knownResourceTypes[rt] = true
}
entries := []annotationsLabelsReportEntry{}
for propertyName := range properties {
if isReservedAnnotationProperty(propertyName) {
continue
}
dot := strings.LastIndex(propertyName, ".")
if dot <= 0 || dot == len(propertyName)-1 {
continue
}
propProcessType := propertyName[:dot]
propResourceType := propertyName[dot+1:]
if !knownResourceTypes[propResourceType] {
continue
}
if processType != "" && propProcessType != processType {
continue
}
if resourceType != "" && propResourceType != resourceType {
continue
}
annotationMap, err := getAnnotation(appName, propProcessType, propResourceType)
if err != nil {
return nil, fmt.Errorf("Unable to read annotation %s: %w", propertyName, err)
}
for key, value := range annotationMap {
entries = append(entries, annotationsLabelsReportEntry{
ProcessType: propProcessType,
ResourceType: propResourceType,
Key: key,
Value: value,
})
}
}
return entries, nil
return collectMetadataEntries(annotationsField, appName, processType, resourceType, getAnnotation)
}
// collectLabelsEntries scans the property store for label entries on appName,
// optionally filtered by processType/resourceType.
func collectLabelsEntries(appName string, processType string, resourceType string) ([]annotationsLabelsReportEntry, error) {
properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, "labels.")
if err != nil {
return nil, fmt.Errorf("Unable to get property list: %w", err)
}
return collectMetadataEntries(labelsField, appName, processType, resourceType, getLabel)
}
knownResourceTypes := map[string]bool{}
for _, rt := range LabelResourceTypes {
knownResourceTypes[rt] = true
// collectMetadataEntries flattens every stored scope a field owns on appName into
// report entries, reading the scopes through the same scan the clear commands use.
func collectMetadataEntries(field metadataField, appName string, processType string, resourceType string, read func(string, string, string) (map[string]string, error)) ([]annotationsLabelsReportEntry, error) {
properties, err := matchingMetadataProperties(field, appName, processType, resourceType)
if err != nil {
return nil, err
}
entries := []annotationsLabelsReportEntry{}
for propertyName := range properties {
suffix := strings.TrimPrefix(propertyName, "labels.")
dot := strings.LastIndex(suffix, ".")
if dot <= 0 || dot == len(suffix)-1 {
continue
}
propProcessType := suffix[:dot]
propResourceType := suffix[dot+1:]
if !knownResourceTypes[propResourceType] {
continue
}
if processType != "" && propProcessType != processType {
continue
}
if resourceType != "" && propResourceType != resourceType {
continue
}
labelMap, err := getLabel(appName, propProcessType, propResourceType)
for _, property := range properties {
values, err := read(appName, property.ProcessType, property.ResourceType)
if err != nil {
return nil, fmt.Errorf("Unable to read label %s: %w", propertyName, err)
return nil, fmt.Errorf("Unable to read %s %s: %w", field.Singular, property.PropertyName, err)
}
for key, value := range labelMap {
for key, value := range values {
entries = append(entries, annotationsLabelsReportEntry{
ProcessType: propProcessType,
ResourceType: propResourceType,
ProcessType: property.ProcessType,
ResourceType: property.ResourceType,
Key: key,
Value: value,
})
@@ -17,9 +17,12 @@ Manage scheduler-k3s settings for an app
Additional commands:`
helpContent = `
scheduler-k3s:annotations:clear <app|--global> [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Clear all annotations for an app or a single process-type/resource-type scope
scheduler-k3s:annotations:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear an annotation for a given app/process-type/resource-type combination
scheduler-k3s:annotations:set --replace <app|--global> <key=value> [<key=value> ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Replace the entire annotation map for a given app/process-type/resource-type combination
scheduler-k3s:annotations:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s annotations report for one or more apps
scheduler-k3s:autoscaling-auth:set <app|--global> <trigger> [<--metadata key=value>...], Set or clear a scheduler-k3s autoscaling keda trigger authentication resource for an app
scheduler-k3s:autoscaling-auth:set --replace <app|--global> <trigger> <--metadata key=value> [<--metadata key=value>...], Replace the entire metadata map for a scheduler-k3s autoscaling keda trigger authentication resource
scheduler-k3s:autoscaling-auth:report [<app>|--global] [--format stdout|json] [--include-metadata], Displays a scheduler-k3s autoscaling auth report for one or more apps
scheduler-k3s:charts:report [<chart>] [--format stdout|json], Displays a scheduler-k3s chart override report
scheduler-k3s:charts:set <chart-name.property> (<value>), Set or clear a chart-specific helm value
@@ -28,9 +31,13 @@ Additional commands:`
scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster
scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster
scheduler-k3s:labels:clear <app|--global> [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Clear all labels for an app or a single process-type/resource-type scope
scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
scheduler-k3s:labels:set --replace <app|--global> <key=value> [<key=value> ...] [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Replace the entire label map for a given app/process-type/resource-type combination
scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
scheduler-k3s:node-sysctls:clear [--global|--profile PROFILE], Clear all node-level kernel sysctls for unprofiled nodes or a single node profile
scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE], Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
scheduler-k3s:node-sysctls:set --replace <sysctl=value> [<sysctl=value> ...] [--global|--profile PROFILE], Replace the entire node-level kernel sysctl map for unprofiled nodes or a single node profile
scheduler-k3s:node-sysctls:report [--format stdout|json], Displays the node-level kernel sysctls applied to each scope
scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster
@@ -18,22 +18,50 @@ func main() {
var err error
switch subcommand {
case "annotations:clear":
args := flag.NewFlagSet("scheduler-k3s:annotations:clear", flag.ExitOnError)
global := args.Bool("global", false, "--global: clear global annotations")
processType := args.String("process-type", "", "--process-type: filter by process-type")
resourceType := args.String("resource-type", "", "--resource-type: filter by resource-type")
args.Parse(os.Args[2:])
appName := args.Arg(0)
if *global {
appName = "--global"
}
err = scheduler_k3s.CommandAnnotationsClear(scheduler_k3s.MetadataClearInput{
AppName: appName,
ProcessType: *processType,
ResourceType: *resourceType,
})
case "annotations:set":
args := flag.NewFlagSet("scheduler-k3s:annotations:set", flag.ExitOnError)
global := args.Bool("global", false, "--global: set a global property")
processType := args.String("process-type", "", "--process-type: scope to process-type")
resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type")
replace := args.Bool("replace", false, "--replace: replace the entire annotation map for the scope")
args.Parse(os.Args[2:])
appName := args.Arg(0)
property := args.Arg(1)
value := args.Arg(2)
if *global {
appName = "--global"
property = args.Arg(0)
value = args.Arg(1)
input := scheduler_k3s.MetadataSetInput{
AppName: args.Arg(0),
ProcessType: *processType,
ResourceType: *resourceType,
Replace: *replace,
}
err = scheduler_k3s.CommandAnnotationsSet(appName, *processType, *resourceType, property, value)
skip := 1
if *global {
input.AppName = "--global"
skip = 0
}
if *replace {
input.Pairs = common.VarArgs(args.Args(), skip)
} else {
input.Key = args.Arg(skip)
input.Value = args.Arg(skip + 1)
}
err = scheduler_k3s.CommandAnnotationsSet(input)
case "annotations:report":
args := flag.NewFlagSet("scheduler-k3s:annotations:report", flag.ExitOnError)
global := args.Bool("global", false, "--global: show the global report")
@@ -55,6 +83,7 @@ func main() {
args := flag.NewFlagSet("scheduler-k3s:autoscaling-auth:set", flag.ExitOnError)
global := args.Bool("global", false, "--global: set a global property")
metadata := args.StringToString("metadata", map[string]string{}, "--metadata: a key=value map of parameter metadata")
replace := args.Bool("replace", false, "--replace: replace the entire metadata map for the trigger")
args.Parse(os.Args[2:])
appName := args.Arg(0)
trigger := args.Arg(1)
@@ -62,7 +91,7 @@ func main() {
appName = "--global"
trigger = args.Arg(0)
}
err = scheduler_k3s.CommandAutoscalingAuthSet(appName, trigger, *metadata, *global)
err = scheduler_k3s.CommandAutoscalingAuthSet(appName, trigger, *metadata, *global, *replace)
case "autoscaling-auth:report":
args := flag.NewFlagSet("scheduler-k3s:autoscaling-auth:report", flag.ExitOnError)
global := args.Bool("global", false, "--global: show the global report")
@@ -131,22 +160,50 @@ func main() {
kubeletArgs := args.StringSlice("kubelet-args", []string{}, "kubelet-args: repeatable key=value kubelet arguments (e.g., --kubelet-args key=value)")
args.Parse(os.Args[2:])
err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling, *kubeletArgs)
case "labels:clear":
args := flag.NewFlagSet("scheduler-k3s:labels:clear", flag.ExitOnError)
global := args.Bool("global", false, "--global: clear global labels")
processType := args.String("process-type", "", "--process-type: filter by process-type")
resourceType := args.String("resource-type", "", "--resource-type: filter by resource-type")
args.Parse(os.Args[2:])
appName := args.Arg(0)
if *global {
appName = "--global"
}
err = scheduler_k3s.CommandLabelsClear(scheduler_k3s.MetadataClearInput{
AppName: appName,
ProcessType: *processType,
ResourceType: *resourceType,
})
case "labels:set":
args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError)
global := args.Bool("global", false, "--global: set a global property")
processType := args.String("process-type", "", "--process-type: scope to process-type")
resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type")
replace := args.Bool("replace", false, "--replace: replace the entire label map for the scope")
args.Parse(os.Args[2:])
appName := args.Arg(0)
property := args.Arg(1)
value := args.Arg(2)
if *global {
appName = "--global"
property = args.Arg(0)
value = args.Arg(1)
input := scheduler_k3s.MetadataSetInput{
AppName: args.Arg(0),
ProcessType: *processType,
ResourceType: *resourceType,
Replace: *replace,
}
err = scheduler_k3s.CommandLabelsSet(appName, *processType, *resourceType, property, value)
skip := 1
if *global {
input.AppName = "--global"
skip = 0
}
if *replace {
input.Pairs = common.VarArgs(args.Args(), skip)
} else {
input.Key = args.Arg(skip)
input.Value = args.Arg(skip + 1)
}
err = scheduler_k3s.CommandLabelsSet(input)
case "labels:report":
args := flag.NewFlagSet("scheduler-k3s:labels:report", flag.ExitOnError)
global := args.Bool("global", false, "--global: show the global report")
@@ -164,18 +221,39 @@ func main() {
appName = "--global"
}
err = scheduler_k3s.CommandLabelsReport(appName, *format, *processType, *resourceType, infoFlag)
case "node-sysctls:set":
args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError)
case "node-sysctls:clear":
args := flag.NewFlagSet("scheduler-k3s:node-sysctls:clear", flag.ExitOnError)
global := args.Bool("global", false, "--global: scope to all nodes without a node profile")
profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes")
args.Parse(os.Args[2:])
key := args.Arg(0)
value := args.Arg(1)
if *global && *profileName != "" {
err = fmt.Errorf("Only one of --global and --profile may be specified")
break
}
err = scheduler_k3s.CommandNodeSysctlsSet(*profileName, key, value)
err = scheduler_k3s.CommandNodeSysctlsClear(*profileName)
case "node-sysctls:set":
args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError)
global := args.Bool("global", false, "--global: scope to all nodes without a node profile")
profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes")
replace := args.Bool("replace", false, "--replace: replace the entire sysctl map for the scope")
args.Parse(os.Args[2:])
if *global && *profileName != "" {
err = fmt.Errorf("Only one of --global and --profile may be specified")
break
}
input := scheduler_k3s.NodeSysctlsSetInput{
ProfileName: *profileName,
Replace: *replace,
}
if *replace {
input.Pairs = args.Args()
} else {
input.Key = args.Arg(0)
input.Value = args.Arg(1)
}
err = scheduler_k3s.CommandNodeSysctlsSet(input)
case "node-sysctls:report":
args := flag.NewFlagSet("scheduler-k3s:node-sysctls:report", flag.ExitOnError)
format := args.String("format", "stdout", "format: [ stdout | json ]")
+155 -79
View File
@@ -20,56 +20,70 @@ import (
"helm.sh/helm/v3/pkg/chartutil"
)
// CommandAnnotationsSet set or clear a scheduler-k3s annotation for an app
func CommandAnnotationsSet(appName string, processType string, resourceType string, key string, value string) error {
if resourceType == "" {
return fmt.Errorf("Missing resource-type")
}
if processType == "" {
processType = GlobalProcessType
}
property := fmt.Sprintf("%s.%s", processType, resourceType)
if value == "" {
if err := common.PropertyMapDelete("scheduler-k3s", appName, property, key); err != nil {
return fmt.Errorf("Unable to delete property map entry: %w", err)
}
return nil
}
if err := common.PropertyMapSet("scheduler-k3s", appName, property, key, value); err != nil {
return fmt.Errorf("Unable to set property map entry: %w", err)
}
return nil
// CommandAnnotationsSet sets, clears or replaces scheduler-k3s annotations for an app
func CommandAnnotationsSet(input MetadataSetInput) error {
return commandMetadataSet(annotationsField, input)
}
// CommandNodeSysctlsSet sets or clears a node-level kernel sysctl for a scope
func CommandNodeSysctlsSet(profileName string, key string, value string) error {
if key == "" {
// CommandAnnotationsClear clears scheduler-k3s annotations for an app
func CommandAnnotationsClear(input MetadataClearInput) error {
return commandMetadataClear(annotationsField, input)
}
// NodeSysctlsSetInput captures the inputs accepted by node-sysctls:set.
// Key/Value and Pairs are mutually exclusive: Pairs is populated only under
// --replace, where the positional arguments are key=value pairs rather than a
// single sysctl name and value.
type NodeSysctlsSetInput struct {
ProfileName string
Key string
Value string
Pairs []string
Replace bool
}
// CommandNodeSysctlsSet sets, clears or replaces node-level kernel sysctls for a scope
func CommandNodeSysctlsSet(input NodeSysctlsSetInput) error {
if err := verifyNodeSysctlsScope(input.ProfileName); err != nil {
return err
}
property := getNodeSysctlsProperty(input.ProfileName)
if input.Replace {
if len(input.Pairs) == 0 {
return fmt.Errorf("Must specify at least one key=value pair, use scheduler-k3s:node-sysctls:clear to remove all sysctls")
}
parsed, err := parseMetadataPairs(input.Pairs)
if err != nil {
return err
}
removed, err := removedSysctlNames(property, parsed)
if err != nil {
return err
}
if err := common.PropertyMapWrite("scheduler-k3s", "--global", property, parsed); err != nil {
return fmt.Errorf("Unable to write property map: %w", err)
}
warnUnmanagedSysctls(removed)
return CreateOrUpdateNodeSysctls(context.Background())
}
if input.Key == "" {
return fmt.Errorf("Missing sysctl name")
}
if profileName != "" {
if err := verifyNodeProfileExists(profileName); err != nil {
return err
}
if err := verifyNodeProfileSysctlsSupported(profileName); err != nil {
return err
}
}
property := getNodeSysctlsProperty(profileName)
if value == "" {
if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, key); err != nil {
if input.Value == "" {
if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, input.Key); err != nil {
return fmt.Errorf("Unable to delete property map entry: %w", err)
}
common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", key))
warnUnmanagedSysctls([]string{input.Key})
} else {
if err := common.PropertyMapSet("scheduler-k3s", "--global", property, key, value); err != nil {
if err := common.PropertyMapSet("scheduler-k3s", "--global", property, input.Key, input.Value); err != nil {
return fmt.Errorf("Unable to set property map entry: %w", err)
}
}
@@ -77,6 +91,68 @@ func CommandNodeSysctlsSet(profileName string, key string, value string) error {
return CreateOrUpdateNodeSysctls(context.Background())
}
// CommandNodeSysctlsClear clears every node-level kernel sysctl for a scope
func CommandNodeSysctlsClear(profileName string) error {
if err := verifyNodeSysctlsScope(profileName); err != nil {
return err
}
property := getNodeSysctlsProperty(profileName)
removed, err := removedSysctlNames(property, map[string]string{})
if err != nil {
return err
}
if err := common.PropertyDelete("scheduler-k3s", "--global", property); err != nil {
return fmt.Errorf("Unable to delete property: %w", err)
}
warnUnmanagedSysctls(removed)
return CreateOrUpdateNodeSysctls(context.Background())
}
// verifyNodeSysctlsScope returns an error when a profile-scoped command names a profile
// that does not exist or cannot back a helm release.
func verifyNodeSysctlsScope(profileName string) error {
if profileName == "" {
return nil
}
if err := verifyNodeProfileExists(profileName); err != nil {
return err
}
return verifyNodeProfileSysctlsSupported(profileName)
}
// removedSysctlNames returns the sysctls a scope currently manages that the next
// stored map no longer will, so they can be warned about before the write happens.
func removedSysctlNames(property string, next map[string]string) ([]string, error) {
current, err := common.PropertyMapGet("scheduler-k3s", "--global", property)
if err != nil {
return nil, fmt.Errorf("Unable to get property map: %w", err)
}
removed := []string{}
for key := range current {
if _, ok := next[key]; !ok {
removed = append(removed, key)
}
}
sort.Strings(removed)
return removed, nil
}
// warnUnmanagedSysctls notes that dokku has stopped managing a sysctl. The last
// value written stays on affected nodes until they reboot, which is how sysctl -w
// behaves everywhere else.
func warnUnmanagedSysctls(names []string) {
for _, name := range names {
common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", name))
}
}
// CommandNodeSysctlsReport displays the configured node-level kernel sysctls
func CommandNodeSysctlsReport(format string) error {
if format != "stdout" && format != "json" {
@@ -149,8 +225,8 @@ func verifyNodeProfileSysctlsSupported(profileName string) error {
return nil
}
// CommandAutoscalingAuthSet set or clear a scheduler-k3s autoscaling keda trigger authentication object for an app
func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool) error {
// CommandAutoscalingAuthSet sets, clears or replaces a scheduler-k3s autoscaling keda trigger authentication object for an app
func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool, replace bool) error {
if global {
appName = "--global"
}
@@ -165,18 +241,17 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri
return fmt.Errorf("Missing trigger type argument")
}
if replace && len(metadata) == 0 {
return fmt.Errorf("Must specify at least one --metadata flag, omit --replace to remove all metadata for the trigger")
}
if len(metadata) == 0 || replace {
if err := deleteTriggerAuthProperties(appName, trigger); err != nil {
return err
}
}
if len(metadata) == 0 {
properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, fmt.Sprintf("%s%s.", TriggerAuthPropertyPrefix, trigger))
if err != nil {
return fmt.Errorf("Unable to get property list: %w", err)
}
for key := range properties {
if err := common.PropertyDelete("scheduler-k3s", appName, key); err != nil {
return fmt.Errorf("Unable to delete property: %w", err)
}
}
if appName == "--global" {
helmAgent, err := NewHelmAgent("keda", DeployLogPrinter)
if err != nil {
@@ -196,11 +271,11 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri
if err := common.PropertyWrite("scheduler-k3s", appName, fmt.Sprintf("%s%s.%s", TriggerAuthPropertyPrefix, trigger, key), value); err != nil {
return fmt.Errorf("Unable to set property: %w", err)
}
common.LogInfo1("Trigger authentication settings saved")
common.LogVerbose("Resources will be created or updated on next deploy")
}
common.LogInfo1("Trigger authentication settings saved")
common.LogVerbose("Resources will be created or updated on next deploy")
if appName == "--global" {
err := applyKedaClusterTriggerAuthentications(context.Background(), trigger, metadata)
if err != nil {
@@ -211,6 +286,22 @@ func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[stri
return nil
}
// deleteTriggerAuthProperties removes every stored metadata key for one trigger.
func deleteTriggerAuthProperties(appName string, trigger string) error {
properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, fmt.Sprintf("%s%s.", TriggerAuthPropertyPrefix, trigger))
if err != nil {
return fmt.Errorf("Unable to get property list: %w", err)
}
for key := range properties {
if err := common.PropertyDelete("scheduler-k3s", appName, key); err != nil {
return fmt.Errorf("Unable to delete property: %w", err)
}
}
return nil
}
// CommandAutoscalingAuthReport displays a scheduler-k3s autoscaling keda trigger authentication report for one or more apps
func CommandAutoscalingAuthReport(appName string, format string, includeMetadata bool, infoFlag string) error {
if len(appName) == 0 {
@@ -1237,29 +1328,14 @@ func CommandEnsureCharts(forceInstall bool, forceChartNames []string) error {
return nil
}
// CommandLabelsSet set or clear a scheduler-k3s label for an app
func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error {
if resourceType == "" {
return fmt.Errorf("Missing resource-type")
}
// CommandLabelsSet sets, clears or replaces scheduler-k3s labels for an app
func CommandLabelsSet(input MetadataSetInput) error {
return commandMetadataSet(labelsField, input)
}
if processType == "" {
processType = GlobalProcessType
}
property := fmt.Sprintf("labels.%s.%s", processType, resourceType)
if value == "" {
if err := common.PropertyMapDelete("scheduler-k3s", appName, property, key); err != nil {
return fmt.Errorf("Unable to delete property map entry: %w", err)
}
return nil
}
if err := common.PropertyMapSet("scheduler-k3s", appName, property, key, value); err != nil {
return fmt.Errorf("Unable to set property map entry: %w", err)
}
return nil
// CommandLabelsClear clears scheduler-k3s labels for an app
func CommandLabelsClear(input MetadataClearInput) error {
return commandMetadataClear(labelsField, input)
}
// CommandProfilesAdd adds a node profile to the k3s cluster