Merge pull request #6595 from dokku/cluster-config

Add ability for users to specify alternative kubeconfig and kubecontext

docs/deployment/schedulers/k3s.md

@@ -320,6 +320,38 @@ By default, Dokku assumes that all it controls all actions on the cluster, and t
 dokku scheduler-k3s:show-kubeconfig
 ```
 
+### Interacting with an external Kubernetes cluster
+
+While the k3s scheduler plugin is designed to work with a Dokku-managed k3s cluster, Dokku can be configured to interact with any Kubernetes cluster by setting the global `kubeconfig-path` to a path to a custom kubeconfig on the Dokku server. This property is only available at a global level.
+
+```shell
+dokku scheduler-k3s:set --global kubeconfig-path /path/to/custom/kubeconfig
+```
+
+To set the default value, omit the value from the `scheduler-k3s:set` call:
+
+```shell
+dokku scheduler-k3s:set --global kubeconfig-path
+```
+
+The default value for the `kubeconfig-path` is the k3s kubeconfig located at `/etc/rancher/k3s/k3s.yaml`.
+
+### Customizing the Kubernetes context
+
+When interacting with a custom Kubeconfig, the `kube-context` property can be set to specify a specific context within the kubeconfig to use. This property is available only at the global leve.
+
+```shell
+dokku scheduler-k3s:set --global kube-context lollipop
+```
+
+To set the default value, omit the value from the `scheduler-k3s:set` call:
+
+```shell
+dokku scheduler-k3s:set --global kube-context
+```
+
+The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig.
+
 ## Scheduler Interface
 
 The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler.

plugins/scheduler-k3s/functions.go

@@ -1142,18 +1142,38 @@ func installHelm(ctx context.Context) error {
 	return nil
 }
 
+// isKubernetesAvailable returns an error if kubernetes api is not available
+func isKubernetesAvailable() error {
+	client, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	if err := client.Ping(); err != nil {
+		return fmt.Errorf("Error pinging kubernetes: %w", err)
+	}
+
+	return nil
+}
+
+// isK3sInstalled returns an error if k3s is not installed
 func isK3sInstalled() error {
 	if !common.FileExists("/usr/local/bin/k3s") {
 		return fmt.Errorf("k3s binary is not available")
 	}
 
-	if !common.FileExists(KubeConfigPath) {
+	if !common.FileExists(getKubeconfigPath()) {
 		return fmt.Errorf("k3s kubeconfig is not available")
 	}
 
 	return nil
 }
 
+// isK3sKubernetes returns true if the current kubernetes cluster is configured to be k3s
+func isK3sKubernetes() bool {
+	return getKubeconfigPath() == KubeConfigPath
+}
+
 func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
 	return func(ctx context.Context) (bool, error) {
 		fmt.Printf(".")

plugins/scheduler-k3s/helm.go

@@ -75,7 +75,9 @@ func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error)
 		helmDriver = "secrets"
 	}
 
-	kubeConfig := kube.GetConfig(KubeConfigPath, "", namespace)
+	kubeconfigPath := getKubeconfigPath()
+	kubeContext := getKubeContext()
+	kubeConfig := kube.GetConfig(kubeconfigPath, kubeContext, namespace)
 	if err := actionConfig.Init(kubeConfig, namespace, helmDriver, logger); err != nil {
 		return nil, err
 	}

plugins/scheduler-k3s/k8s.go

@@ -22,11 +22,22 @@ import (
 	"k8s.io/utils/ptr"
 )
 
+func getKubeconfigPath() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "kubeconfig-path", KubeConfigPath)
+}
+
+func getKubeContext() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "kube-context", DefaultKubeContext)
+}
+
 // KubernetesClient is a wrapper around the Kubernetes client
 type KubernetesClient struct {
 	// Client is the Kubernetes client
 	Client kubernetes.Clientset
 
+	// KubeConfigPath is the path to the Kubernetes config
+	KubeConfigPath string
+
 	// RestClient is the Kubernetes REST client
 	RestClient rest.Interface
 
@@ -36,7 +47,9 @@ type KubernetesClient struct {
 
 // NewKubernetesClient creates a new Kubernetes client
 func NewKubernetesClient() (KubernetesClient, error) {
-	clientConfig := KubernetesClientConfig()
+	kubeconfigPath := getKubeconfigPath()
+	kubeContext := getKubeContext()
+	clientConfig := KubernetesClientConfig(kubeconfigPath, kubeContext)
 	restConf, err := clientConfig.ClientConfig()
 	if err != nil {
 		return KubernetesClient{}, err
@@ -60,17 +73,29 @@ func NewKubernetesClient() (KubernetesClient, error) {
 	}
 
 	return KubernetesClient{
-		Client:     *client,
+		Client:         *client,
-		RestConfig: *restConf,
+		KubeConfigPath: kubeconfigPath,
-		RestClient: restClient,
+		RestConfig:     *restConf,
+		RestClient:     restClient,
 	}, nil
 }
 
 // KubernetesClientConfig returns a Kubernetes client config
-func KubernetesClientConfig() clientcmd.ClientConfig {
+func KubernetesClientConfig(kubeconfigPath string, kubecontext string) clientcmd.ClientConfig {
+	configOverrides := clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}
+	if kubecontext != "" {
+		configOverrides.CurrentContext = kubecontext
+	}
+
 	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		&clientcmd.ClientConfigLoadingRules{ExplicitPath: KubeConfigPath},
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
-		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
+		&configOverrides,
+	)
+}
+
+func (k KubernetesClient) Ping() error {
+	_, err := k.Client.Discovery().ServerVersion()
+	return err
 }
 
 // AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
@@ -110,16 +135,23 @@ type ApplyKubernetesManifestInput struct {
 }
 
 func (k KubernetesClient) ApplyKubernetesManifest(ctx context.Context, input ApplyKubernetesManifestInput) error {
+	args := []string{
+		"apply",
+		"-f",
+		input.Manifest,
+	}
+
+	if kubeContext := getKubeContext(); kubeContext != "" {
+		args = append([]string{"--context", kubeContext}, args...)
+	}
+
+	if kubeconfigPath := getKubeconfigPath(); kubeconfigPath != "" {
+		args = append([]string{"--kubeconfig", kubeconfigPath}, args...)
+	}
+
 	upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{
-		Command: "kubectl",
+		Command:     "kubectl",
-		Args: []string{
+		Args:        args,
-			"apply",
-			"-f",
-			input.Manifest,
-		},
-		Env: map[string]string{
-			"KUBECONFIG": KubeConfigPath,
-		},
 		StreamStdio: true,
 	})
 	if err != nil {

plugins/scheduler-k3s/report.go

@@ -17,6 +17,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		"--scheduler-k3s-computed-image-pull-secrets":   reportComputedImagePullSecrets,
 		"--scheduler-k3s-image-pull-secrets":            reportImagePullSecrets,
 		"--scheduler-k3s-global-image-pull-secrets":     reportGlobalImagePullSecrets,
+		"--scheduler-k3s-global-kubeconfig-path":        reportGlobalKubeconfigPath,
+		"--scheduler-k3s-global-kube-context":           reportGlobalKubeContext,
 		"--scheduler-k3s-computed-letsencrypt-server":   reportComputedLetsencryptServer,
 		"--scheduler-k3s-letsencrypt-server":            reportLetsencryptServer,
 		"--scheduler-k3s-global-letsencrypt-server":     reportGlobalLetsencryptServer,
@@ -71,6 +73,13 @@ func reportGlobalIngressClass(appName string) string {
 	return getGlobalIngressClass()
 }
 
+func reportGlobalKubeconfigPath(appName string) string {
+	return getKubeconfigPath()
+}
+
+func reportGlobalKubeContext(appName string) string {
+	return getKubeContext()
+}
 func reportComputedLetsencryptServer(appName string) string {
 	return getComputedLetsencryptServer(appName)
 }

plugins/scheduler-k3s/scheduler_k3s.go

@@ -29,6 +29,8 @@ var (
 		"deploy-timeout":         true,
 		"image-pull-secrets":     true,
 		"ingress-class":          true,
+		"kube-context":           true,
+		"kubeconfig-path":        true,
 		"letsencrypt-server":     true,
 		"letsencrypt-email-prod": true,
 		"letsencrypt-email-stag": true,
@@ -42,6 +44,7 @@ var (
 const DefaultIngressClass = "traefik"
 const GlobalProcessType = "--global"
 const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
+const DefaultKubeContext = ""
 
 var (
 	runtimeScheme  = runtime.NewScheme()

plugins/scheduler-k3s/subcommands.go

@@ -315,7 +315,16 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
 // CommandClusterAdd adds a server to the k3s cluster
 func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUknownHosts bool, taintScheduling bool) error {
 	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot join cluster")
+		return fmt.Errorf("k3s not installed, cannot add node to cluster: %w", err)
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available, cannot add node to cluster: %w", err)
 	}
 
 	if role != "server" && role != "worker" {
@@ -530,11 +539,6 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn
 		return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
 	}
 
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Unable to create kubernetes client: %w", err)
-	}
-
 	common.LogInfo2Quiet("Waiting for node to exist")
 	nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
 		Clientset:  clientset,
@@ -588,9 +592,6 @@ func CommandClusterList(format string) error {
 	if format != "stdout" && format != "json" {
 		return fmt.Errorf("Invalid format: %s", format)
 	}
-	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot list cluster nodes")
-	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
@@ -608,6 +609,10 @@ func CommandClusterList(format string) error {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available, cannot list cluster nodes: %w", err)
+	}
+
 	nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
 	if err != nil {
 		return fmt.Errorf("Unable to list nodes: %w", err)
@@ -641,7 +646,7 @@ func CommandClusterList(format string) error {
 // CommandClusterRemove removes a node from the k3s cluster
 func CommandClusterRemove(nodeName string) error {
 	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot remove node")
+		return fmt.Errorf("k3s not installed, cannot remove node from cluster: %w", err)
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -661,6 +666,10 @@ func CommandClusterRemove(nodeName string) error {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	common.LogVerboseQuiet("Getting node remote connection information")
 	node, err := clientset.GetNode(ctx, GetNodeInput{
 		Name: nodeName,
@@ -730,11 +739,12 @@ func CommandSet(appName string, property string, value string) error {
 
 // CommandShowKubeconfig displays the kubeconfig file contents
 func CommandShowKubeconfig() error {
-	if !common.FileExists(KubeConfigPath) {
+	kubeconfigPath := getKubeconfigPath()
-		return fmt.Errorf("Kubeconfig file does not exist: %s", KubeConfigPath)
+	if !common.FileExists(kubeconfigPath) {
+		return fmt.Errorf("Kubeconfig file does not exist: %s", kubeconfigPath)
 	}
 
-	b, err := os.ReadFile(KubeConfigPath)
+	b, err := os.ReadFile(kubeconfigPath)
 	if err != nil {
 		return fmt.Errorf("Unable to read kubeconfig file: %w", err)
 	}
@@ -746,7 +756,7 @@ func CommandShowKubeconfig() error {
 
 func CommandUninstall() error {
 	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot uninstall")
+		return fmt.Errorf("k3s not installed, cannot uninstall: %w", err)
 	}
 
 	common.LogInfo1("Uninstalling k3s")

plugins/scheduler-k3s/triggers.go

@@ -414,6 +414,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
 		Namespace:     namespace,
@@ -573,6 +577,10 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	namespace := getComputedNamespace(appName)
 	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 1
@@ -664,6 +672,10 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 0
 	if processType != "" {
@@ -926,6 +938,10 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -1078,6 +1094,10 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	namespace := getComputedNamespace(appName)
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
@@ -1139,9 +1159,26 @@ func TriggerSchedulerPostDelete(scheduler string, appName string) error {
 		return nil
 	}
 
-	if err := isK3sInstalled(); err != nil {
+	dataErr := common.RemoveAppDataDirectory("logs", appName)
-		common.LogWarn(fmt.Sprintf("Skipping app deletion: %s", err.Error()))
+	propertyErr := common.PropertyDestroy("logs", appName)
-		return nil
+
+	if dataErr != nil {
+		return dataErr
+	}
+
+	if propertyErr != nil {
+		return propertyErr
+	}
+
+	if isK3sKubernetes() {
+		if err := isK3sInstalled(); err != nil {
+			common.LogWarn("k3s is not installed, skipping")
+			return nil
+		}
+	}
+
+	if err := isKubernetesAvailable(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
 	}
 
 	namespace := getComputedNamespace(appName)
@@ -1164,11 +1201,6 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 		return nil
 	}
 
-	if err := isK3sInstalled(); err != nil {
-		common.LogWarn(fmt.Sprintf("Skipping app stop: %s", err.Error()))
-		return nil
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -1182,9 +1214,19 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 
 	clientset, err := NewKubernetesClient()
 	if err != nil {
+		if isK3sKubernetes() {
+			if err := isK3sInstalled(); err != nil {
+				common.LogWarn("k3s is not installed, skipping")
+				return nil
+			}
+		}
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	namespace := getComputedNamespace(appName)
 	deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
 		Namespace:     namespace,