feat: label nodes with the node profile they joined with
Node profiles controlled how a node joined the cluster but left no trace on the node afterwards, so a profile could not be selected against with `kubectl`, a `nodeSelector`, or a node affinity rule. Nodes joined without a profile are left unlabeled, and the server node created by `scheduler-k3s:initialize` never carries the label since it does not pass through `scheduler-k3s:cluster:add`.
This commit is contained in:
@@ -214,6 +214,21 @@ dokku scheduler-k3s:profiles:remove edge-workers
|
|||||||
|
|
||||||
Removal only deletes the stored definition; nodes that already joined the cluster keep their existing configuration.
|
Removal only deletes the stored definition; nodes that already joined the cluster keep their existing configuration.
|
||||||
|
|
||||||
|
#### The node profile label
|
||||||
|
|
||||||
|
When a node joins via `scheduler-k3s:cluster:add --profile <name>`, Dokku labels it with `dokku.com/node-profile=<name>`. This makes a profile selectable after the fact, whether via `kubectl`, a `nodeSelector`, or a node affinity rule.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
kubectl get nodes -L dokku.com/node-profile
|
||||||
|
```
|
||||||
|
|
||||||
|
Nodes added without `--profile` are not labeled, as an empty label value would be indistinguishable from a profile literally named the empty string.
|
||||||
|
|
||||||
|
Two limits are worth knowing before relying on this label:
|
||||||
|
|
||||||
|
- The server node never carries it. That node is created by `scheduler-k3s:initialize` and never passes through `scheduler-k3s:cluster:add`, so no profile is ever associated with it.
|
||||||
|
- Nodes that joined before this label existed are not backfilled. Use `kubectl label node <node> dokku.com/node-profile=<name>` to set it on an existing node.
|
||||||
|
|
||||||
### Changing deployment settings
|
### Changing deployment settings
|
||||||
|
|
||||||
The k3s plugin provides a number of settings that can be used to managed deployments on a per-app basis. The following table outlines ones not covered elsewhere:
|
The k3s plugin provides a number of settings that can be used to managed deployments on a per-app basis. The following table outlines ones not covered elsewhere:
|
||||||
|
|||||||
@@ -1286,6 +1286,27 @@ func resolveLetsencryptIssuer(appName string, clusterIssuerName string, appEmail
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nodeLabels returns the labels to apply to a node joining the cluster, including
|
||||||
|
// the node profile label when the node was added with a named profile. The returned
|
||||||
|
// map is always a fresh copy so callers cannot mutate ServerLabels or WorkerLabels.
|
||||||
|
func nodeLabels(role string, profileName string) map[string]string {
|
||||||
|
source := ServerLabels
|
||||||
|
if role == "worker" {
|
||||||
|
source = WorkerLabels
|
||||||
|
}
|
||||||
|
|
||||||
|
labels := make(map[string]string, len(source)+1)
|
||||||
|
for key, value := range source {
|
||||||
|
labels[key] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
if profileName != "" {
|
||||||
|
labels[NodeProfileLabel] = profileName
|
||||||
|
}
|
||||||
|
|
||||||
|
return labels
|
||||||
|
}
|
||||||
|
|
||||||
// InitializeInstallerArgsInput contains the inputs to initializeInstallerArgs
|
// InitializeInstallerArgsInput contains the inputs to initializeInstallerArgs
|
||||||
type InitializeInstallerArgsInput struct {
|
type InitializeInstallerArgsInput struct {
|
||||||
// IngressClass is the ingress class the cluster is initialized with
|
// IngressClass is the ingress class the cluster is initialized with
|
||||||
|
|||||||
@@ -178,3 +178,80 @@ func containsArgPair(args []string, flag string, value string) bool {
|
|||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNodeLabels(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
role string
|
||||||
|
profileName string
|
||||||
|
wantKey string
|
||||||
|
wantValue string
|
||||||
|
wantProfile bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "server without a profile",
|
||||||
|
role: "server",
|
||||||
|
profileName: "",
|
||||||
|
wantKey: "svccontroller.k3s.cattle.io/enablelb",
|
||||||
|
wantValue: "true",
|
||||||
|
wantProfile: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "worker without a profile",
|
||||||
|
role: "worker",
|
||||||
|
profileName: "",
|
||||||
|
wantKey: "node-role.kubernetes.io/worker",
|
||||||
|
wantValue: "worker",
|
||||||
|
wantProfile: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "worker with a profile",
|
||||||
|
role: "worker",
|
||||||
|
profileName: "edge-workers",
|
||||||
|
wantKey: "node-role.kubernetes.io/worker",
|
||||||
|
wantValue: "worker",
|
||||||
|
wantProfile: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "server with a profile",
|
||||||
|
role: "server",
|
||||||
|
profileName: "control-plane",
|
||||||
|
wantKey: "svccontroller.k3s.cattle.io/enablelb",
|
||||||
|
wantValue: "true",
|
||||||
|
wantProfile: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := nodeLabels(tc.role, tc.profileName)
|
||||||
|
|
||||||
|
if got[tc.wantKey] != tc.wantValue {
|
||||||
|
t.Errorf("nodeLabels() role label = %q, want %q", got[tc.wantKey], tc.wantValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
profile, ok := got[NodeProfileLabel]
|
||||||
|
if ok != tc.wantProfile {
|
||||||
|
t.Errorf("nodeLabels() has %s = %v, want %v", NodeProfileLabel, ok, tc.wantProfile)
|
||||||
|
}
|
||||||
|
if tc.wantProfile && profile != tc.profileName {
|
||||||
|
t.Errorf("nodeLabels() %s = %q, want %q", NodeProfileLabel, profile, tc.profileName)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeLabelsDoesNotMutatePackageLabels(t *testing.T) {
|
||||||
|
serverBefore := len(ServerLabels)
|
||||||
|
workerBefore := len(WorkerLabels)
|
||||||
|
|
||||||
|
nodeLabels("server", "control-plane")
|
||||||
|
nodeLabels("worker", "edge-workers")
|
||||||
|
|
||||||
|
if len(ServerLabels) != serverBefore {
|
||||||
|
t.Errorf("nodeLabels() mutated ServerLabels: len = %d, want %d", len(ServerLabels), serverBefore)
|
||||||
|
}
|
||||||
|
if len(WorkerLabels) != workerBefore {
|
||||||
|
t.Errorf("nodeLabels() mutated WorkerLabels: len = %d, want %d", len(WorkerLabels), workerBefore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -231,6 +231,9 @@ type NodeProfile struct {
|
|||||||
KubeletArgs []string `json:"kubelet_args,omitempty"`
|
KubeletArgs []string `json:"kubelet_args,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NodeProfileLabel is the node label recording the node profile a node was added with
|
||||||
|
const NodeProfileLabel = "dokku.com/node-profile"
|
||||||
|
|
||||||
// ServerLabels are the labels for a server node
|
// ServerLabels are the labels for a server node
|
||||||
var ServerLabels = map[string]string{
|
var ServerLabels = map[string]string{
|
||||||
"svccontroller.k3s.cattle.io/enablelb": "true",
|
"svccontroller.k3s.cattle.io/enablelb": "true",
|
||||||
|
|||||||
@@ -875,10 +875,7 @@ export INSTALL_K3S_VERSION=%s
|
|||||||
return fmt.Errorf("Unable to find node after joining cluster, node will not be annotated/labeled appropriately access registry secrets")
|
return fmt.Errorf("Unable to find node after joining cluster, node will not be annotated/labeled appropriately access registry secrets")
|
||||||
}
|
}
|
||||||
|
|
||||||
labels := ServerLabels
|
labels := nodeLabels(incomingProfile.Role, profileName)
|
||||||
if incomingProfile.Role == "worker" {
|
|
||||||
labels = WorkerLabels
|
|
||||||
}
|
|
||||||
|
|
||||||
for key, value := range labels {
|
for key, value := range labels {
|
||||||
common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))
|
common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))
|
||||||
|
|||||||
Reference in New Issue
Block a user