fix: respect the k3s master node version when adding a new worker to the k3s cluster
Previously, it was possible to have a kubernetes version mismatch on new worker nodes if a newer version of kubernetes was released prior to upgrading the master nodes to that version. This can sometimes cause cluster instability. This change fetches the lowest node version associated with a master node and then uses that to install k3s on the worker node.
This commit is contained in:
2 files changed
+93
-1
No files matched your search
@@ -14,6 +14,7 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/Masterminds/semver/v3"
|
||||||
"github.com/dokku/dokku/plugins/common"
|
"github.com/dokku/dokku/plugins/common"
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
"github.com/go-openapi/jsonpointer"
|
"github.com/go-openapi/jsonpointer"
|
||||||
@@ -139,6 +140,42 @@ func (k KubernetesClient) Ping() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (k KubernetesClient) GetLowestNodeVersion(ctx context.Context, input ListNodesInput) (string, error) {
|
||||||
|
nodes, err := k.ListNodes(ctx, input)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to list nodes: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return "", fmt.Errorf("no nodes found in the cluster")
|
||||||
|
}
|
||||||
|
|
||||||
|
var lowestVersion *semver.Version
|
||||||
|
for _, node := range nodes {
|
||||||
|
kubeletVersion := node.Status.NodeInfo.KubeletVersion
|
||||||
|
if kubeletVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
versionStr := strings.TrimPrefix(kubeletVersion, "v")
|
||||||
|
version, err := semver.NewVersion(versionStr)
|
||||||
|
if err != nil {
|
||||||
|
common.LogWarn(fmt.Sprintf("Failed to parse version %s for node %s: %v", kubeletVersion, node.Name, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if lowestVersion == nil || version.LessThan(lowestVersion) {
|
||||||
|
lowestVersion = version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if lowestVersion == nil {
|
||||||
|
return "", fmt.Errorf("no valid kubelet versions found")
|
||||||
|
}
|
||||||
|
|
||||||
|
return "v" + lowestVersion.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
// AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
|
// AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
|
||||||
type AnnotateNodeInput struct {
|
type AnnotateNodeInput struct {
|
||||||
// Name is the Kubernetes node name
|
// Name is the Kubernetes node name
|
||||||
|
|||||||
@@ -551,6 +551,61 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn
|
|||||||
return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
|
return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
common.LogInfo2Quiet("Ensuring compatible k3s version for node")
|
||||||
|
lowestNodeVersion, err := clientset.GetLowestNodeVersion(ctx, ListNodesInput{
|
||||||
|
LabelSelector: "node-role.kubernetes.io/master=true",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to get lowest node version: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpFile, err := os.CreateTemp("", "k3s-installer-*.sh")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create temporary file: %w", err)
|
||||||
|
}
|
||||||
|
defer os.Remove(tmpFile.Name())
|
||||||
|
|
||||||
|
scriptContent := fmt.Sprintf(`#!/usr/bin/env bash
|
||||||
|
set -x
|
||||||
|
export INSTALL_K3S_VERSION=%s
|
||||||
|
|
||||||
|
/tmp/k3s-installer.sh "$@"`, lowestNodeVersion)
|
||||||
|
|
||||||
|
if _, err := tmpFile.WriteString(scriptContent); err != nil {
|
||||||
|
return fmt.Errorf("failed to write to temporary file: %w", err)
|
||||||
|
}
|
||||||
|
tmpFile.Close()
|
||||||
|
|
||||||
|
sftpCopyCmd, err := common.CallSftpCopy(common.SftpCopyInput{
|
||||||
|
AllowUknownHosts: allowUknownHosts,
|
||||||
|
DestinationPath: "/tmp/k3s-installer-executor.sh",
|
||||||
|
RemoteHost: remoteHost,
|
||||||
|
SourcePath: tmpFile.Name(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to copy installer script via sftp: %w", err)
|
||||||
|
}
|
||||||
|
if sftpCopyCmd.ExitErr != nil {
|
||||||
|
return fmt.Errorf("Invalid exit code from sftp copy command: %d", sftpCopyCmd.ExitErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
chmodExecutorCmd, err := common.CallSshCommand(common.SshCommandInput{
|
||||||
|
Command: "chmod",
|
||||||
|
Args: []string{
|
||||||
|
"0755",
|
||||||
|
"/tmp/k3s-installer-executor.sh",
|
||||||
|
},
|
||||||
|
AllowUknownHosts: allowUknownHosts,
|
||||||
|
RemoteHost: remoteHost,
|
||||||
|
StreamStdio: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to make installer script executable via ssh: %w", err)
|
||||||
|
}
|
||||||
|
if chmodExecutorCmd.ExitCode != 0 {
|
||||||
|
return fmt.Errorf("Invalid exit code from chmod command via ssh: %d", chmodExecutorCmd.ExitCode)
|
||||||
|
}
|
||||||
|
|
||||||
u, err := url.Parse(remoteHost)
|
u, err := url.Parse(remoteHost)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to parse remote host: %w", err)
|
return fmt.Errorf("failed to parse remote host: %w", err)
|
||||||
@@ -612,7 +667,7 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn
|
|||||||
|
|
||||||
common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
|
common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
|
||||||
joinCmd, err := common.CallSshCommand(common.SshCommandInput{
|
joinCmd, err := common.CallSshCommand(common.SshCommandInput{
|
||||||
Command: "/tmp/k3s-installer.sh",
|
Command: "/tmp/k3s-installer-executor.sh",
|
||||||
Args: args,
|
Args: args,
|
||||||
AllowUknownHosts: allowUknownHosts,
|
AllowUknownHosts: allowUknownHosts,
|
||||||
RemoteHost: remoteHost,
|
RemoteHost: remoteHost,
|
||||||
|
|||||||
Reference in new issue
Block a user