diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index edd7329dc..5ce122cd9 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -8,7 +8,9 @@ require ( github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000 github.com/fatih/color v1.16.0 + github.com/go-openapi/jsonpointer v0.19.6 github.com/go-resty/resty/v2 v2.11.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/rancher/wharfie v0.6.5 github.com/ryanuber/columnize v2.1.2+incompatible @@ -66,7 +68,6 @@ require ( github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gobwas/glob v0.2.3 // indirect @@ -84,7 +85,6 @@ require ( github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index 49accdbeb..2ccfee834 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -73,6 +73,37 @@ func KubernetesClientConfig() clientcmd.ClientConfig { &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}) } +// AnnotateNodeInput contains all the information needed to annotates a Kubernetes node +type AnnotateNodeInput struct { + // Name is the Kubernetes node name + Name string + // Key is the annotation key + Key string + // Value is the annotation value + Value string +} + +// AnnotateNode annotates a Kubernetes node +func (k KubernetesClient) AnnotateNode(ctx context.Context, input AnnotateNodeInput) error { + node, err := k.Client.CoreV1().Nodes().Get(ctx, input.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if node == nil { + return errors.New("node is nil") + } + + keyPath := fmt.Sprintf("/metadata/annotations/%s", jsonpointer.Escape(input.Key)) + patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value) + _, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("failed to annotate node: %w", err) + } + + return nil +} + // CreateJobInput contains all the information needed to create a Kubernetes job type CreateJobInput struct { // Job is the Kubernetes job @@ -190,9 +221,9 @@ func (k KubernetesClient) LabelNode(ctx context.Context, input LabelNodeInput) e return errors.New("node is nil") } - keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape("kubernetes.io/role")) - patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, "worker") - _, err = k.Client.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) + keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape(input.Key)) + patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value) + _, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) if err != nil { return fmt.Errorf("failed to label node: %w", err) } diff --git a/plugins/scheduler-k3s/registry.go b/plugins/scheduler-k3s/registry.go new file mode 100644 index 000000000..824e1b7b4 --- /dev/null +++ b/plugins/scheduler-k3s/registry.go @@ -0,0 +1,78 @@ +package scheduler_k3s + +import ( + "context" + "fmt" + + "github.com/dokku/dokku/plugins/common" +) + +func copyRegistryToNode(ctx context.Context, remoteHost string) error { + common.LogInfo1Quiet(fmt.Sprintf("Updating k3s registry configuration on %s", remoteHost)) + sftpCmd, err := common.CallSftpCopy(common.SftpCopyInput{ + AllowUknownHosts: true, + DestinationPath: "/tmp/registries.yaml", + RemoteHost: remoteHost, + SourcePath: RegistryConfigPath, + }) + if err != nil { + return fmt.Errorf("Error copying registries.yaml to remote host: %w", err) + } + + if sftpCmd.ExitErr != nil { + return fmt.Errorf("Error copying registries.yaml to remote host: %s", sftpCmd.ExitErr.Error()) + } + + common.LogVerboseQuiet("Moving k3s registry configuration into place") + mvCmd, err := common.CallSshCommand(common.SshCommandInput{ + AllowUknownHosts: true, + Args: []string{"/tmp/registries.yaml", RegistryConfigPath}, + Command: "mv", + RemoteHost: remoteHost, + StreamStdio: true, + Sudo: true, + }) + if err != nil { + return fmt.Errorf("Unable to call mv command over ssh: %w", err) + } + + if mvCmd.ExitCode != 0 { + return fmt.Errorf("Invalid exit code from mv command over ssh: %d", mvCmd.ExitCode) + } + + common.LogVerboseQuiet("Updating k3s registry permissions") + chmodCmd, err := common.CallSshCommand(common.SshCommandInput{ + AllowUknownHosts: true, + Args: []string{"0644", RegistryConfigPath}, + Command: "chmod", + RemoteHost: remoteHost, + StreamStdio: true, + Sudo: true, + }) + if err != nil { + return fmt.Errorf("Unable to call chmod command over ssh: %w", err) + } + + if chmodCmd.ExitCode != 0 { + return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode) + } + + common.LogVerboseQuiet("Updating k3s registry ower") + chownCmd, err := common.CallSshCommand(common.SshCommandInput{ + AllowUknownHosts: true, + Args: []string{"root:root", RegistryConfigPath}, + Command: "chown", + RemoteHost: remoteHost, + StreamStdio: true, + Sudo: true, + }) + if err != nil { + return fmt.Errorf("Unable to call chown command over ssh: %w", err) + } + + if chownCmd.ExitCode != 0 { + return fmt.Errorf("Invalid exit code from chown command over ssh: %d", chownCmd.ExitCode) + } + + return nil +} diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 64c045fce..31060faf8 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -462,7 +462,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule") } - common.LogInfo2Quiet("Joining k3s cluster") + common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName)) joinCmd, err := common.CallSshCommand(common.SshCommandInput{ Command: "/tmp/k3s-installer.sh", Args: args, @@ -477,27 +477,31 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta if joinCmd.ExitCode != 0 { return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode) } + ctx := context.Background() + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Unable to create kubernetes client: %w", err) + } + + common.LogInfo2Quiet("Waiting for node to exist") + nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{ + Clientset: clientset, + NodeName: nodeName, + RetryCount: 20, + }) + if err != nil { + return fmt.Errorf("Error waiting for pod to exist: %w", err) + } + if len(nodes) == 0 { + return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled appropriately and cannot access registry secrets") + } + + err = copyRegistryToNode(ctx, remoteHost) + if err != nil { + return fmt.Errorf("Error copying registries.yaml to remote host: %w", err) + } if role == "worker" { - ctx := context.Background() - clientset, err := NewKubernetesClient() - if err != nil { - return fmt.Errorf("Unable to create kubernetes client: %w", err) - } - - common.LogInfo2Quiet("Waiting for node to exist") - nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{ - Clientset: clientset, - NodeName: nodeName, - RetryCount: 20, - }) - if err != nil { - return fmt.Errorf("Error waiting for pod to exist: %w", err) - } - if len(nodes) == 0 { - return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled kubernetes.io/role=worker") - } - common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker") err = clientset.LabelNode(ctx, LabelNodeInput{ Name: nodes[0].Name, @@ -509,6 +513,16 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta } } + common.LogInfo2Quiet("Annotating node with connection information") + err = clientset.AnnotateNode(ctx, AnnotateNodeInput{ + Name: nodes[0].Name, + Key: "dokku.com/remote-host", + Value: remoteHost, + }) + if err != nil { + return fmt.Errorf("Unable to patch node: %w", err) + } + common.LogVerboseQuiet("Done") return nil } diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 0b2e9d3e2..4c0368c50 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -21,6 +21,7 @@ import ( "github.com/dokku/dokku/plugins/config" "github.com/dokku/dokku/plugins/cron" "github.com/fatih/color" + "github.com/hashicorp/go-multierror" "github.com/kballard/go-shellquote" "github.com/rancher/wharfie/pkg/registries" "github.com/ryanuber/columnize" @@ -121,7 +122,33 @@ func TriggerPostRegistryLogin(server string, username string) error { } // todo: auth against all nodes in cluster - return nil + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Error creating kubernetes client: %w", err) + } + + ctx := context.Background() + nodes, err := clientset.ListNodes(ctx, ListNodesInput{}) + if err != nil { + return fmt.Errorf("Error listing nodes: %w", err) + } + + var result error + for _, node := range nodes { + remoteHost, ok := node.Annotations["dokku.com/remote-host"] + if !ok { + continue + } + + err := copyRegistryToNode(ctx, remoteHost) + if err != nil { + wrappedErr := fmt.Errorf("Error copying registry to node: %w", err) + result = multierror.Append(result, wrappedErr) + common.LogWarn(wrappedErr.Error()) + } + } + + return result } // TriggerSchedulerDeploy deploys an image tag for a given application