feat: label nodes with the node profile they joined with

Node profiles controlled how a node joined the cluster but left no trace on the node afterwards, so a profile could not be selected against with `kubectl`, a `nodeSelector`, or a node affinity rule. Nodes joined without a profile are left unlabeled, and the server node created by `scheduler-k3s:initialize` never carries the label since it does not pass through `scheduler-k3s:cluster:add`.

docs/deployment/schedulers/k3s.md

@@ -214,6 +214,21 @@ dokku scheduler-k3s:profiles:remove edge-workers
 
 Removal only deletes the stored definition; nodes that already joined the cluster keep their existing configuration.
 
+#### The node profile label
+
+When a node joins via `scheduler-k3s:cluster:add --profile <name>`, Dokku labels it with `dokku.com/node-profile=<name>`. This makes a profile selectable after the fact, whether via `kubectl`, a `nodeSelector`, or a node affinity rule.
+
+```shell
+kubectl get nodes -L dokku.com/node-profile
+```
+
+Nodes added without `--profile` are not labeled, as an empty label value would be indistinguishable from a profile literally named the empty string.
+
+Two limits are worth knowing before relying on this label:
+
+- The server node never carries it. That node is created by `scheduler-k3s:initialize` and never passes through `scheduler-k3s:cluster:add`, so no profile is ever associated with it.
+- Nodes that joined before this label existed are not backfilled. Use `kubectl label node <node> dokku.com/node-profile=<name>` to set it on an existing node.
+
 ### Changing deployment settings
 
 The k3s plugin provides a number of settings that can be used to managed deployments on a per-app basis. The following table outlines ones not covered elsewhere:

plugins/scheduler-k3s/functions.go

@@ -1286,6 +1286,27 @@ func resolveLetsencryptIssuer(appName string, clusterIssuerName string, appEmail
 	}
 }
 
+// nodeLabels returns the labels to apply to a node joining the cluster, including
+// the node profile label when the node was added with a named profile. The returned
+// map is always a fresh copy so callers cannot mutate ServerLabels or WorkerLabels.
+func nodeLabels(role string, profileName string) map[string]string {
+	source := ServerLabels
+	if role == "worker" {
+		source = WorkerLabels
+	}
+
+	labels := make(map[string]string, len(source)+1)
+	for key, value := range source {
+		labels[key] = value
+	}
+
+	if profileName != "" {
+		labels[NodeProfileLabel] = profileName
+	}
+
+	return labels
+}
+
 // InitializeInstallerArgsInput contains the inputs to initializeInstallerArgs
 type InitializeInstallerArgsInput struct {
 	// IngressClass is the ingress class the cluster is initialized with

plugins/scheduler-k3s/functions_test.go

@@ -178,3 +178,80 @@ func containsArgPair(args []string, flag string, value string) bool {
 	}
 	return false
 }
+
+func TestNodeLabels(t *testing.T) {
+	cases := []struct {
+		name        string
+		role        string
+		profileName string
+		wantKey     string
+		wantValue   string
+		wantProfile bool
+	}{
+		{
+			name:        "server without a profile",
+			role:        "server",
+			profileName: "",
+			wantKey:     "svccontroller.k3s.cattle.io/enablelb",
+			wantValue:   "true",
+			wantProfile: false,
+		},
+		{
+			name:        "worker without a profile",
+			role:        "worker",
+			profileName: "",
+			wantKey:     "node-role.kubernetes.io/worker",
+			wantValue:   "worker",
+			wantProfile: false,
+		},
+		{
+			name:        "worker with a profile",
+			role:        "worker",
+			profileName: "edge-workers",
+			wantKey:     "node-role.kubernetes.io/worker",
+			wantValue:   "worker",
+			wantProfile: true,
+		},
+		{
+			name:        "server with a profile",
+			role:        "server",
+			profileName: "control-plane",
+			wantKey:     "svccontroller.k3s.cattle.io/enablelb",
+			wantValue:   "true",
+			wantProfile: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := nodeLabels(tc.role, tc.profileName)
+
+			if got[tc.wantKey] != tc.wantValue {
+				t.Errorf("nodeLabels() role label = %q, want %q", got[tc.wantKey], tc.wantValue)
+			}
+
+			profile, ok := got[NodeProfileLabel]
+			if ok != tc.wantProfile {
+				t.Errorf("nodeLabels() has %s = %v, want %v", NodeProfileLabel, ok, tc.wantProfile)
+			}
+			if tc.wantProfile && profile != tc.profileName {
+				t.Errorf("nodeLabels() %s = %q, want %q", NodeProfileLabel, profile, tc.profileName)
+			}
+		})
+	}
+}
+
+func TestNodeLabelsDoesNotMutatePackageLabels(t *testing.T) {
+	serverBefore := len(ServerLabels)
+	workerBefore := len(WorkerLabels)
+
+	nodeLabels("server", "control-plane")
+	nodeLabels("worker", "edge-workers")
+
+	if len(ServerLabels) != serverBefore {
+		t.Errorf("nodeLabels() mutated ServerLabels: len = %d, want %d", len(ServerLabels), serverBefore)
+	}
+	if len(WorkerLabels) != workerBefore {
+		t.Errorf("nodeLabels() mutated WorkerLabels: len = %d, want %d", len(WorkerLabels), workerBefore)
+	}
+}

plugins/scheduler-k3s/scheduler_k3s.go

@@ -231,6 +231,9 @@ type NodeProfile struct {
 	KubeletArgs []string `json:"kubelet_args,omitempty"`
 }
 
+// NodeProfileLabel is the node label recording the node profile a node was added with
+const NodeProfileLabel = "dokku.com/node-profile"
+
 // ServerLabels are the labels for a server node
 var ServerLabels = map[string]string{
 	"svccontroller.k3s.cattle.io/enablelb": "true",

plugins/scheduler-k3s/subcommands.go

@@ -875,10 +875,7 @@ export INSTALL_K3S_VERSION=%s
 		return fmt.Errorf("Unable to find node after joining cluster, node will not be annotated/labeled appropriately access registry secrets")
 	}
 
-	labels := ServerLabels
-	if incomingProfile.Role == "worker" {
-		labels = WorkerLabels
-	}
+	labels := nodeLabels(incomingProfile.Role, profileName)
 
 	for key, value := range labels {
 		common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))