Merge pull request #8031 from dokku/pin-k3s-version

Respect the k3s master node version when adding a new worker to the k3s cluster

plugins/scheduler-k3s/k8s.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/fatih/color"
 	"github.com/go-openapi/jsonpointer"
@@ -139,6 +140,42 @@ func (k KubernetesClient) Ping() error {
 	return err
 }
 
+func (k KubernetesClient) GetLowestNodeVersion(ctx context.Context, input ListNodesInput) (string, error) {
+	nodes, err := k.ListNodes(ctx, input)
+	if err != nil {
+		return "", fmt.Errorf("failed to list nodes: %w", err)
+	}
+
+	if len(nodes) == 0 {
+		return "", fmt.Errorf("no nodes found in the cluster")
+	}
+
+	var lowestVersion *semver.Version
+	for _, node := range nodes {
+		kubeletVersion := node.Status.NodeInfo.KubeletVersion
+		if kubeletVersion == "" {
+			continue
+		}
+
+		versionStr := strings.TrimPrefix(kubeletVersion, "v")
+		version, err := semver.NewVersion(versionStr)
+		if err != nil {
+			common.LogWarn(fmt.Sprintf("Failed to parse version %s for node %s: %v", kubeletVersion, node.Name, err))
+			continue
+		}
+
+		if lowestVersion == nil || version.LessThan(lowestVersion) {
+			lowestVersion = version
+		}
+	}
+
+	if lowestVersion == nil {
+		return "", fmt.Errorf("no valid kubelet versions found")
+	}
+
+	return "v" + lowestVersion.String(), nil
+}
+
 // AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
 type AnnotateNodeInput struct {
 	// Name is the Kubernetes node name

plugins/scheduler-k3s/subcommands.go

@@ -551,6 +551,61 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn
 		return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
 	}
 
+	common.LogInfo2Quiet("Ensuring compatible k3s version for node")
+	lowestNodeVersion, err := clientset.GetLowestNodeVersion(ctx, ListNodesInput{
+		LabelSelector: "node-role.kubernetes.io/master=true",
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to get lowest node version: %w", err)
+	}
+
+	tmpFile, err := os.CreateTemp("", "k3s-installer-*.sh")
+	if err != nil {
+		return fmt.Errorf("failed to create temporary file: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	scriptContent := fmt.Sprintf(`#!/usr/bin/env bash
+set -x
+export INSTALL_K3S_VERSION=%s
+
+/tmp/k3s-installer.sh "$@"`, lowestNodeVersion)
+
+	if _, err := tmpFile.WriteString(scriptContent); err != nil {
+		return fmt.Errorf("failed to write to temporary file: %w", err)
+	}
+	tmpFile.Close()
+
+	sftpCopyCmd, err := common.CallSftpCopy(common.SftpCopyInput{
+		AllowUknownHosts: allowUknownHosts,
+		DestinationPath:  "/tmp/k3s-installer-executor.sh",
+		RemoteHost:       remoteHost,
+		SourcePath:       tmpFile.Name(),
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to copy installer script via sftp: %w", err)
+	}
+	if sftpCopyCmd.ExitErr != nil {
+		return fmt.Errorf("Invalid exit code from sftp copy command: %d", sftpCopyCmd.ExitErr)
+	}
+
+	chmodExecutorCmd, err := common.CallSshCommand(common.SshCommandInput{
+		Command: "chmod",
+		Args: []string{
+			"0755",
+			"/tmp/k3s-installer-executor.sh",
+		},
+		AllowUknownHosts: allowUknownHosts,
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to make installer script executable via ssh: %w", err)
+	}
+	if chmodExecutorCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from chmod command via ssh: %d", chmodExecutorCmd.ExitCode)
+	}
+
 	u, err := url.Parse(remoteHost)
 	if err != nil {
 		return fmt.Errorf("failed to parse remote host: %w", err)
@@ -612,7 +667,7 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn
 
 	common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
 	joinCmd, err := common.CallSshCommand(common.SshCommandInput{
-		Command:          "/tmp/k3s-installer.sh",
+		Command:          "/tmp/k3s-installer-executor.sh",
 		Args:             args,
 		AllowUknownHosts: allowUknownHosts,
 		RemoteHost:       remoteHost,