From 72067fcfd2d072be280490f01f5dc4cbfe39e4e6 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 14 Feb 2024 03:50:15 -0500 Subject: [PATCH] feat: add ability for users to specify alternative kubeconfig and kubecontext This will provide the possibility for users to talk to existing kubernetes clusters, thereby removing one of the biggest reasons for having the old scheduler-kubernetes plugin around. --- docs/deployment/schedulers/k3s.md | 32 +++++++++++++ plugins/scheduler-k3s/functions.go | 15 +++++- plugins/scheduler-k3s/helm.go | 4 +- plugins/scheduler-k3s/k8s.go | 64 +++++++++++++++++++------- plugins/scheduler-k3s/report.go | 9 ++++ plugins/scheduler-k3s/scheduler_k3s.go | 3 ++ plugins/scheduler-k3s/subcommands.go | 38 +++++++++------ plugins/scheduler-k3s/triggers.go | 34 ++++++++++---- 8 files changed, 159 insertions(+), 40 deletions(-) diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index a63ca3e3a..a85c189a6 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -320,6 +320,38 @@ By default, Dokku assumes that all it controls all actions on the cluster, and t dokku scheduler-k3s:show-kubeconfig ``` +### Interacting with an external Kubernetes cluster + +While the k3s scheduler plugin is designed to work with a Dokku-managed k3s cluster, Dokku can be configured to interact with any Kubernetes cluster by setting the global `kubeconfig-path` to a path to a custom kubeconfig on the Dokku server. This property is only available at a global level. + +```shell +dokku scheduler-k3s:set --global kubeconfig-path /path/to/custom/kubeconfig +``` + +To set the default value, omit the value from the `scheduler-k3s:set` call: + +```shell +dokku scheduler-k3s:set --global kubeconfig-path +``` + +The default value for the `kubeconfig-path` is the k3s kubeconfig located at `/etc/rancher/k3s/k3s.yaml`. + +### Customizing the Kubernetes context + +When interacting with a custom Kubeconfig, the `kube-context` property can be set to specify a specific context within the kubeconfig to use. This property is available only at the global leve. + +```shell +dokku scheduler-k3s:set --global kube-context lollipop +``` + +To set the default value, omit the value from the `scheduler-k3s:set` call: + +```shell +dokku scheduler-k3s:set --global kube-context +``` + +The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig. + ## Scheduler Interface The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler. diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 365d1268d..ad9e775ad 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1142,12 +1142,25 @@ func installHelm(ctx context.Context) error { return nil } +func isKubernetesAvailable() error { + client, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Error creating kubernetes client: %w", err) + } + + if err := client.Ping(); err != nil { + return fmt.Errorf("Error pinging kubernetes: %w", err) + } + + return nil +} + func isK3sInstalled() error { if !common.FileExists("/usr/local/bin/k3s") { return fmt.Errorf("k3s binary is not available") } - if !common.FileExists(KubeConfigPath) { + if !common.FileExists(getKubeconfigPath()) { return fmt.Errorf("k3s kubeconfig is not available") } diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index bf16fa1dc..875296105 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -75,7 +75,9 @@ func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error) helmDriver = "secrets" } - kubeConfig := kube.GetConfig(KubeConfigPath, "", namespace) + kubeconfigPath := getKubeconfigPath() + kubeContext := getKubeContext() + kubeConfig := kube.GetConfig(kubeconfigPath, kubeContext, namespace) if err := actionConfig.Init(kubeConfig, namespace, helmDriver, logger); err != nil { return nil, err } diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index 99483f883..556d8d4ab 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -22,11 +22,22 @@ import ( "k8s.io/utils/ptr" ) +func getKubeconfigPath() string { + return common.PropertyGetDefault("scheduler-k3s", "--global", "kubeconfig-path", KubeConfigPath) +} + +func getKubeContext() string { + return common.PropertyGetDefault("scheduler-k3s", "--global", "kube-context", DefaultKubeContext) +} + // KubernetesClient is a wrapper around the Kubernetes client type KubernetesClient struct { // Client is the Kubernetes client Client kubernetes.Clientset + // KubeConfigPath is the path to the Kubernetes config + KubeConfigPath string + // RestClient is the Kubernetes REST client RestClient rest.Interface @@ -36,7 +47,9 @@ type KubernetesClient struct { // NewKubernetesClient creates a new Kubernetes client func NewKubernetesClient() (KubernetesClient, error) { - clientConfig := KubernetesClientConfig() + kubeconfigPath := getKubeconfigPath() + kubeContext := getKubeContext() + clientConfig := KubernetesClientConfig(kubeconfigPath, kubeContext) restConf, err := clientConfig.ClientConfig() if err != nil { return KubernetesClient{}, err @@ -60,17 +73,29 @@ func NewKubernetesClient() (KubernetesClient, error) { } return KubernetesClient{ - Client: *client, - RestConfig: *restConf, - RestClient: restClient, + Client: *client, + KubeConfigPath: kubeconfigPath, + RestConfig: *restConf, + RestClient: restClient, }, nil } // KubernetesClientConfig returns a Kubernetes client config -func KubernetesClientConfig() clientcmd.ClientConfig { +func KubernetesClientConfig(kubeconfigPath string, kubecontext string) clientcmd.ClientConfig { + configOverrides := clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}} + if kubecontext != "" { + configOverrides.CurrentContext = kubecontext + } + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: KubeConfigPath}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}) + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &configOverrides, + ) +} + +func (k KubernetesClient) Ping() error { + _, err := k.Client.Discovery().ServerVersion() + return err } // AnnotateNodeInput contains all the information needed to annotates a Kubernetes node @@ -110,16 +135,23 @@ type ApplyKubernetesManifestInput struct { } func (k KubernetesClient) ApplyKubernetesManifest(ctx context.Context, input ApplyKubernetesManifestInput) error { + args := []string{ + "apply", + "-f", + input.Manifest, + } + + if kubeContext := getKubeContext(); kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + + if kubeconfigPath := getKubeconfigPath(); kubeconfigPath != "" { + args = append([]string{"--kubeconfig", kubeconfigPath}, args...) + } + upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{ - Command: "kubectl", - Args: []string{ - "apply", - "-f", - input.Manifest, - }, - Env: map[string]string{ - "KUBECONFIG": KubeConfigPath, - }, + Command: "kubectl", + Args: args, StreamStdio: true, }) if err != nil { diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index 25474883b..2e25445df 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -17,6 +17,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-computed-image-pull-secrets": reportComputedImagePullSecrets, "--scheduler-k3s-image-pull-secrets": reportImagePullSecrets, "--scheduler-k3s-global-image-pull-secrets": reportGlobalImagePullSecrets, + "--scheduler-k3s-global-kubeconfig-path": reportGlobalKubeconfigPath, + "--scheduler-k3s-global-kube-context": reportGlobalKubeContext, "--scheduler-k3s-computed-letsencrypt-server": reportComputedLetsencryptServer, "--scheduler-k3s-letsencrypt-server": reportLetsencryptServer, "--scheduler-k3s-global-letsencrypt-server": reportGlobalLetsencryptServer, @@ -71,6 +73,13 @@ func reportGlobalIngressClass(appName string) string { return getGlobalIngressClass() } +func reportGlobalKubeconfigPath(appName string) string { + return getKubeconfigPath() +} + +func reportGlobalKubeContext(appName string) string { + return getKubeContext() +} func reportComputedLetsencryptServer(appName string) string { return getComputedLetsencryptServer(appName) } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 0aca6c1b3..9064124fc 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -29,6 +29,8 @@ var ( "deploy-timeout": true, "image-pull-secrets": true, "ingress-class": true, + "kube-context": true, + "kubeconfig-path": true, "letsencrypt-server": true, "letsencrypt-email-prod": true, "letsencrypt-email-stag": true, @@ -42,6 +44,7 @@ var ( const DefaultIngressClass = "traefik" const GlobalProcessType = "--global" const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" +const DefaultKubeContext = "" var ( runtimeScheme = runtime.NewScheme() diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 594c938eb..dd85668c7 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -315,7 +315,16 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo // CommandClusterAdd adds a server to the k3s cluster func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUknownHosts bool, taintScheduling bool) error { if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot join cluster") + return fmt.Errorf("k3s not installed, cannot add node to cluster: %w", err) + } + + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Unable to create kubernetes client: %w", err) + } + + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available, cannot add node to cluster: %w", err) } if role != "server" && role != "worker" { @@ -530,11 +539,6 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode) } - clientset, err := NewKubernetesClient() - if err != nil { - return fmt.Errorf("Unable to create kubernetes client: %w", err) - } - common.LogInfo2Quiet("Waiting for node to exist") nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{ Clientset: clientset, @@ -588,9 +592,6 @@ func CommandClusterList(format string) error { if format != "stdout" && format != "json" { return fmt.Errorf("Invalid format: %s", format) } - if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot list cluster nodes") - } ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) @@ -608,6 +609,10 @@ func CommandClusterList(format string) error { return fmt.Errorf("Unable to create kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available, cannot list cluster nodes: %w", err) + } + nodes, err := clientset.ListNodes(ctx, ListNodesInput{}) if err != nil { return fmt.Errorf("Unable to list nodes: %w", err) @@ -641,7 +646,7 @@ func CommandClusterList(format string) error { // CommandClusterRemove removes a node from the k3s cluster func CommandClusterRemove(nodeName string) error { if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot remove node") + return fmt.Errorf("k3s not installed, cannot remove node from cluster: %w", err) } ctx, cancel := context.WithCancel(context.Background()) @@ -661,6 +666,10 @@ func CommandClusterRemove(nodeName string) error { return fmt.Errorf("Unable to create kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + common.LogVerboseQuiet("Getting node remote connection information") node, err := clientset.GetNode(ctx, GetNodeInput{ Name: nodeName, @@ -730,11 +739,12 @@ func CommandSet(appName string, property string, value string) error { // CommandShowKubeconfig displays the kubeconfig file contents func CommandShowKubeconfig() error { - if !common.FileExists(KubeConfigPath) { - return fmt.Errorf("Kubeconfig file does not exist: %s", KubeConfigPath) + kubeconfigPath := getKubeconfigPath() + if !common.FileExists(kubeconfigPath) { + return fmt.Errorf("Kubeconfig file does not exist: %s", kubeconfigPath) } - b, err := os.ReadFile(KubeConfigPath) + b, err := os.ReadFile(kubeconfigPath) if err != nil { return fmt.Errorf("Unable to read kubeconfig file: %w", err) } @@ -746,7 +756,7 @@ func CommandShowKubeconfig() error { func CommandUninstall() error { if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot uninstall") + return fmt.Errorf("k3s not installed, cannot uninstall: %w", err) } common.LogInfo1("Uninstalling k3s") diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index f0578468a..589d73c61 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -414,6 +414,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{ LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), Namespace: namespace, @@ -573,6 +577,10 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string, return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + namespace := getComputedNamespace(appName) labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)} processIndex := 1 @@ -664,6 +672,10 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string, return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)} processIndex := 0 if processType != "" { @@ -926,6 +938,10 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGHUP, @@ -1078,6 +1094,10 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + namespace := getComputedNamespace(appName) cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{ LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), @@ -1139,9 +1159,8 @@ func TriggerSchedulerPostDelete(scheduler string, appName string) error { return nil } - if err := isK3sInstalled(); err != nil { - common.LogWarn(fmt.Sprintf("Skipping app deletion: %s", err.Error())) - return nil + if err := isKubernetesAvailable(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) } namespace := getComputedNamespace(appName) @@ -1164,11 +1183,6 @@ func TriggerSchedulerStop(scheduler string, appName string) error { return nil } - if err := isK3sInstalled(); err != nil { - common.LogWarn(fmt.Sprintf("Skipping app stop: %s", err.Error())) - return nil - } - ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGHUP, @@ -1185,6 +1199,10 @@ func TriggerSchedulerStop(scheduler string, appName string) error { return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + namespace := getComputedNamespace(appName) deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{ Namespace: namespace,