feat: add --kubelet-args to scheduler-k3s:initialize
The server node created by `scheduler-k3s:initialize` had no way to receive kubelet arguments, unlike nodes joined through `scheduler-k3s:cluster:add` or configured via `scheduler-k3s:profiles:add`. This meant settings such as `allowed-unsafe-sysctls` were unreachable on a single-node install.
This commit is contained in:
@@ -78,6 +78,20 @@ Dokku can also use Traefik on cluster initialization via the [Traefik's CRDs](ht
|
|||||||
dokku scheduler-k3s:initialize --ingress-class traefik
|
dokku scheduler-k3s:initialize --ingress-class traefik
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Kubelet flags for the initial server node can be supplied by passing `--kubelet-args` with a comma-separated `key=value` list. This is the only way to configure the kubelet on the node created by `scheduler-k3s:initialize`, as that node never passes through `scheduler-k3s:cluster:add`.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
dokku scheduler-k3s:initialize \
|
||||||
|
--kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6
|
||||||
|
```
|
||||||
|
|
||||||
|
Multiple kubelet arguments can be specified in the same call by separating them with commas.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
dokku scheduler-k3s:initialize \
|
||||||
|
--kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6,max-pods=150
|
||||||
|
```
|
||||||
|
|
||||||
### Adding nodes to the cluster
|
### Adding nodes to the cluster
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
|
|||||||
@@ -1286,6 +1286,65 @@ func resolveLetsencryptIssuer(appName string, clusterIssuerName string, appEmail
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InitializeInstallerArgsInput contains the inputs to initializeInstallerArgs
|
||||||
|
type InitializeInstallerArgsInput struct {
|
||||||
|
// IngressClass is the ingress class the cluster is initialized with
|
||||||
|
IngressClass string
|
||||||
|
// KubeletArgs is a list of key=value kubelet arguments for the server node
|
||||||
|
KubeletArgs []string
|
||||||
|
// NodeName is the generated name of the server node
|
||||||
|
NodeName string
|
||||||
|
// TaintScheduling is whether to taint the node against app workloads
|
||||||
|
TaintScheduling bool
|
||||||
|
// Token is the cluster join token
|
||||||
|
Token string
|
||||||
|
}
|
||||||
|
|
||||||
|
// initializeInstallerArgs builds the argument list handed to the k3s installer
|
||||||
|
// when creating the initial server node
|
||||||
|
func initializeInstallerArgs(input InitializeInstallerArgsInput) []string {
|
||||||
|
args := []string{
|
||||||
|
// initialize the cluster
|
||||||
|
"--cluster-init",
|
||||||
|
// disable local-storage
|
||||||
|
"--disable", "local-storage",
|
||||||
|
// disable traefik so it can be installed separately
|
||||||
|
"--disable", "traefik",
|
||||||
|
// expose etcd metrics
|
||||||
|
"--etcd-expose-metrics",
|
||||||
|
// use wireguard for flannel
|
||||||
|
"--flannel-backend=wireguard-native",
|
||||||
|
// bind controller-manager to all interfaces
|
||||||
|
"--kube-controller-manager-arg", "bind-address=0.0.0.0",
|
||||||
|
// bind proxy metrics to all interfaces
|
||||||
|
"--kube-proxy-arg", "metrics-bind-address=0.0.0.0",
|
||||||
|
// bind scheduler to all interfaces
|
||||||
|
"--kube-scheduler-arg", "bind-address=0.0.0.0",
|
||||||
|
// gc terminated pods
|
||||||
|
"--kube-controller-manager-arg", "terminated-pod-gc-threshold=10",
|
||||||
|
// specify the node name
|
||||||
|
"--node-name", input.NodeName,
|
||||||
|
// allow access for the dokku user
|
||||||
|
"--write-kubeconfig-mode", "0644",
|
||||||
|
// specify a token
|
||||||
|
"--token", input.Token,
|
||||||
|
}
|
||||||
|
|
||||||
|
if input.TaintScheduling {
|
||||||
|
args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, kubeletArg := range input.KubeletArgs {
|
||||||
|
args = append(args, "--kubelet-arg", kubeletArg)
|
||||||
|
}
|
||||||
|
|
||||||
|
if input.IngressClass == "nginx" {
|
||||||
|
args = append(args, "--disable", "traefik")
|
||||||
|
}
|
||||||
|
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
func getKustomizeDirectory(appName string) string {
|
func getKustomizeDirectory(appName string) string {
|
||||||
directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName)
|
directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName)
|
||||||
return filepath.Join(directory, "kustomization")
|
return filepath.Join(directory, "kustomization")
|
||||||
|
|||||||
@@ -91,3 +91,90 @@ func TestNeedsImagePullSecretsPrune(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInitializeInstallerArgs(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
input InitializeInstallerArgsInput
|
||||||
|
wantPairs [][2]string
|
||||||
|
wantAbsent []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no kubelet args",
|
||||||
|
input: InitializeInstallerArgsInput{
|
||||||
|
IngressClass: "traefik",
|
||||||
|
NodeName: "ip-10-0-0-1-abc",
|
||||||
|
Token: "sometoken",
|
||||||
|
},
|
||||||
|
wantAbsent: []string{"--kubelet-arg", "--node-taint"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single kubelet arg",
|
||||||
|
input: InitializeInstallerArgsInput{
|
||||||
|
IngressClass: "traefik",
|
||||||
|
KubeletArgs: []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
|
||||||
|
NodeName: "ip-10-0-0-1-abc",
|
||||||
|
Token: "sometoken",
|
||||||
|
},
|
||||||
|
wantPairs: [][2]string{
|
||||||
|
{"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple kubelet args each get their own flag",
|
||||||
|
input: InitializeInstallerArgsInput{
|
||||||
|
IngressClass: "traefik",
|
||||||
|
KubeletArgs: []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem", "max-pods=150"},
|
||||||
|
NodeName: "ip-10-0-0-1-abc",
|
||||||
|
Token: "sometoken",
|
||||||
|
},
|
||||||
|
wantPairs: [][2]string{
|
||||||
|
{"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
|
||||||
|
{"--kubelet-arg", "max-pods=150"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "kubelet args coexist with taint scheduling",
|
||||||
|
input: InitializeInstallerArgsInput{
|
||||||
|
IngressClass: "nginx",
|
||||||
|
KubeletArgs: []string{"max-pods=150"},
|
||||||
|
NodeName: "ip-10-0-0-1-abc",
|
||||||
|
TaintScheduling: true,
|
||||||
|
Token: "sometoken",
|
||||||
|
},
|
||||||
|
wantPairs: [][2]string{
|
||||||
|
{"--node-taint", "CriticalAddonsOnly=true:NoSchedule"},
|
||||||
|
{"--kubelet-arg", "max-pods=150"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := initializeInstallerArgs(tc.input)
|
||||||
|
|
||||||
|
for _, pair := range tc.wantPairs {
|
||||||
|
if !containsArgPair(got, pair[0], pair[1]) {
|
||||||
|
t.Errorf("initializeInstallerArgs() missing %q %q, got %v", pair[0], pair[1], got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, absent := range tc.wantAbsent {
|
||||||
|
for _, arg := range got {
|
||||||
|
if arg == absent {
|
||||||
|
t.Errorf("initializeInstallerArgs() unexpectedly contains %q, got %v", absent, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsArgPair(args []string, flag string, value string) bool {
|
||||||
|
for i := 0; i < len(args)-1; i++ {
|
||||||
|
if args[i] == flag && args[i+1] == value {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ Additional commands:`
|
|||||||
scheduler-k3s:cluster:list [--format json|stdout], Lists all nodes in a Dokku-managed cluster
|
scheduler-k3s:cluster:list [--format json|stdout], Lists all nodes in a Dokku-managed cluster
|
||||||
scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster
|
scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster
|
||||||
scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
|
scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
|
||||||
scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster
|
scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster
|
||||||
scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
|
scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
|
||||||
scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
|
scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
|
||||||
scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
|
scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
|
||||||
|
|||||||
@@ -128,8 +128,9 @@ func main() {
|
|||||||
taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
|
taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
|
||||||
serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
|
serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
|
||||||
ingressClass := args.String("ingress-class", "nginx", "ingress-class: ingress-class to use for all outbound traffic")
|
ingressClass := args.String("ingress-class", "nginx", "ingress-class: ingress-class to use for all outbound traffic")
|
||||||
|
kubeletArgs := args.StringSlice("kubelet-args", []string{}, "kubelet-args: repeatable key=value kubelet arguments (e.g., --kubelet-args key=value)")
|
||||||
args.Parse(os.Args[2:])
|
args.Parse(os.Args[2:])
|
||||||
err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling)
|
err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling, *kubeletArgs)
|
||||||
case "labels:set":
|
case "labels:set":
|
||||||
args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError)
|
args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError)
|
||||||
global := args.Bool("global", false, "--global: set a global property")
|
global := args.Bool("global", false, "--global: set a global property")
|
||||||
|
|||||||
@@ -174,7 +174,7 @@ func CommandLabelsReport(appName string, format string, processType string, reso
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CommandInitialize initializes a k3s cluster on the local server
|
// CommandInitialize initializes a k3s cluster on the local server
|
||||||
func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error {
|
func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool, kubeletArgs []string) error {
|
||||||
if ingressClass != "nginx" && ingressClass != "traefik" {
|
if ingressClass != "nginx" && ingressClass != "traefik" {
|
||||||
return fmt.Errorf("Invalid ingress-class: %s", ingressClass)
|
return fmt.Errorf("Invalid ingress-class: %s", ingressClass)
|
||||||
}
|
}
|
||||||
@@ -308,40 +308,15 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
|
|||||||
}
|
}
|
||||||
nodeName = strings.ReplaceAll(strings.ToLower(fmt.Sprintf("ip-%s-%s", nodeName, fmt.Sprintf("%X", b))), ".", "-")
|
nodeName = strings.ReplaceAll(strings.ToLower(fmt.Sprintf("ip-%s-%s", nodeName, fmt.Sprintf("%X", b))), ".", "-")
|
||||||
|
|
||||||
args := []string{
|
|
||||||
// initialize the cluster
|
|
||||||
"--cluster-init",
|
|
||||||
// disable local-storage
|
|
||||||
"--disable", "local-storage",
|
|
||||||
// disable traefik so it can be installed separately
|
|
||||||
"--disable", "traefik",
|
|
||||||
// expose etcd metrics
|
|
||||||
"--etcd-expose-metrics",
|
|
||||||
// use wireguard for flannel
|
|
||||||
"--flannel-backend=wireguard-native",
|
|
||||||
// bind controller-manager to all interfaces
|
|
||||||
"--kube-controller-manager-arg", "bind-address=0.0.0.0",
|
|
||||||
// bind proxy metrics to all interfaces
|
|
||||||
"--kube-proxy-arg", "metrics-bind-address=0.0.0.0",
|
|
||||||
// bind scheduler to all interfaces
|
|
||||||
"--kube-scheduler-arg", "bind-address=0.0.0.0",
|
|
||||||
// gc terminated pods
|
|
||||||
"--kube-controller-manager-arg", "terminated-pod-gc-threshold=10",
|
|
||||||
// specify the node name
|
|
||||||
"--node-name", nodeName,
|
|
||||||
// allow access for the dokku user
|
|
||||||
"--write-kubeconfig-mode", "0644",
|
|
||||||
// specify a token
|
|
||||||
"--token", token,
|
|
||||||
}
|
|
||||||
if taintScheduling {
|
|
||||||
args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
|
|
||||||
}
|
|
||||||
|
|
||||||
common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties)
|
common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties)
|
||||||
if ingressClass == "nginx" {
|
|
||||||
args = append(args, "--disable", "traefik")
|
args := initializeInstallerArgs(InitializeInstallerArgsInput{
|
||||||
}
|
IngressClass: ingressClass,
|
||||||
|
KubeletArgs: kubeletArgs,
|
||||||
|
NodeName: nodeName,
|
||||||
|
TaintScheduling: taintScheduling,
|
||||||
|
Token: token,
|
||||||
|
})
|
||||||
|
|
||||||
common.LogInfo2Quiet("Running k3s installer")
|
common.LogInfo2Quiet("Running k3s installer")
|
||||||
installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
|
installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
|
||||||
|
|||||||
@@ -740,6 +740,10 @@ install_k3s() {
|
|||||||
args="$args --server-ip $CI_SERVER_IP"
|
args="$args --server-ip $CI_SERVER_IP"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$KUBELET_ARGS" ]]; then
|
||||||
|
args="$args --kubelet-args $KUBELET_ARGS"
|
||||||
|
fi
|
||||||
|
|
||||||
run /bin/bash -c "dokku scheduler-k3s:initialize ${args}"
|
run /bin/bash -c "dokku scheduler-k3s:initialize ${args}"
|
||||||
echo "output: $output"
|
echo "output: $output"
|
||||||
echo "status: $status"
|
echo "status: $status"
|
||||||
|
|||||||
Reference in New Issue
Block a user