github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: copy the registry file to all remote hosts

Browse Source
This commit is contained in:
Jose Diaz-Gonzalez
2024-01-20 06:57:20 -05:00
parent a354210f67
commit 3375474a4c
5 changed files with 176 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
plugins/scheduler-k3s/go.mod
View File

@@ -8,7 +8,9 @@ require (
github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
github.com/fatih/color v1.16.0
github.com/go-openapi/jsonpointer v0.19.6
github.com/go-resty/resty/v2 v2.11.0
github.com/hashicorp/go-multierror v1.1.1
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/rancher/wharfie v0.6.5
github.com/ryanuber/columnize v2.1.2+incompatible
@@ -66,7 +68,6 @@ require (
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gobwas/glob v0.2.3 // indirect
@@ -84,7 +85,6 @@ require (
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect

37
plugins/scheduler-k3s/k8s.go
View File

@@ -73,6 +73,37 @@ func KubernetesClientConfig() clientcmd.ClientConfig {
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
}
// AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
type AnnotateNodeInput struct {
// Name is the Kubernetes node name
Name string
// Key is the annotation key
Key string
// Value is the annotation value
Value string
}
// AnnotateNode annotates a Kubernetes node
func (k KubernetesClient) AnnotateNode(ctx context.Context, input AnnotateNodeInput) error {
node, err := k.Client.CoreV1().Nodes().Get(ctx, input.Name, metav1.GetOptions{})
if err != nil {
return err
}
if node == nil {
return errors.New("node is nil")
}
keyPath := fmt.Sprintf("/metadata/annotations/%s", jsonpointer.Escape(input.Key))
patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value)
_, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
if err != nil {
return fmt.Errorf("failed to annotate node: %w", err)
}
return nil
}
// CreateJobInput contains all the information needed to create a Kubernetes job
type CreateJobInput struct {
// Job is the Kubernetes job
@@ -190,9 +221,9 @@ func (k KubernetesClient) LabelNode(ctx context.Context, input LabelNodeInput) e
return errors.New("node is nil")
}
keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape("kubernetes.io/role"))
patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, "worker")
_, err = k.Client.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape(input.Key))
patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value)
_, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
if err != nil {
return fmt.Errorf("failed to label node: %w", err)
}

78
plugins/scheduler-k3s/registry.go Normal file
View File

@@ -0,0 +1,78 @@
package scheduler_k3s
import (
"context"
"fmt"
"github.com/dokku/dokku/plugins/common"
)
func copyRegistryToNode(ctx context.Context, remoteHost string) error {
common.LogInfo1Quiet(fmt.Sprintf("Updating k3s registry configuration on %s", remoteHost))
sftpCmd, err := common.CallSftpCopy(common.SftpCopyInput{
AllowUknownHosts: true,
DestinationPath: "/tmp/registries.yaml",
RemoteHost: remoteHost,
SourcePath: RegistryConfigPath,
})
if err != nil {
return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
}
if sftpCmd.ExitErr != nil {
return fmt.Errorf("Error copying registries.yaml to remote host: %s", sftpCmd.ExitErr.Error())
}
common.LogVerboseQuiet("Moving k3s registry configuration into place")
mvCmd, err := common.CallSshCommand(common.SshCommandInput{
AllowUknownHosts: true,
Args: []string{"/tmp/registries.yaml", RegistryConfigPath},
Command: "mv",
RemoteHost: remoteHost,
StreamStdio: true,
Sudo: true,
})
if err != nil {
return fmt.Errorf("Unable to call mv command over ssh: %w", err)
}
if mvCmd.ExitCode != 0 {
return fmt.Errorf("Invalid exit code from mv command over ssh: %d", mvCmd.ExitCode)
}
common.LogVerboseQuiet("Updating k3s registry permissions")
chmodCmd, err := common.CallSshCommand(common.SshCommandInput{
AllowUknownHosts: true,
Args: []string{"0644", RegistryConfigPath},
Command: "chmod",
RemoteHost: remoteHost,
StreamStdio: true,
Sudo: true,
})
if err != nil {
return fmt.Errorf("Unable to call chmod command over ssh: %w", err)
}
if chmodCmd.ExitCode != 0 {
return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
}
common.LogVerboseQuiet("Updating k3s registry ower")
chownCmd, err := common.CallSshCommand(common.SshCommandInput{
AllowUknownHosts: true,
Args: []string{"root:root", RegistryConfigPath},
Command: "chown",
RemoteHost: remoteHost,
StreamStdio: true,
Sudo: true,
})
if err != nil {
return fmt.Errorf("Unable to call chown command over ssh: %w", err)
}
if chownCmd.ExitCode != 0 {
return fmt.Errorf("Invalid exit code from chown command over ssh: %d", chownCmd.ExitCode)
}
return nil
}

54
plugins/scheduler-k3s/subcommands.go
View File

@@ -462,7 +462,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
}
common.LogInfo2Quiet("Joining k3s cluster")
common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
joinCmd, err := common.CallSshCommand(common.SshCommandInput{
Command: "/tmp/k3s-installer.sh",
Args: args,
@@ -477,27 +477,31 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
if joinCmd.ExitCode != 0 {
return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
}
ctx := context.Background()
clientset, err := NewKubernetesClient()
if err != nil {
return fmt.Errorf("Unable to create kubernetes client: %w", err)
}
common.LogInfo2Quiet("Waiting for node to exist")
nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
Clientset: clientset,
NodeName: nodeName,
RetryCount: 20,
})
if err != nil {
return fmt.Errorf("Error waiting for pod to exist: %w", err)
}
if len(nodes) == 0 {
return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled appropriately and cannot access registry secrets")
}
err = copyRegistryToNode(ctx, remoteHost)
if err != nil {
return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
}
if role == "worker" {
ctx := context.Background()
clientset, err := NewKubernetesClient()
if err != nil {
return fmt.Errorf("Unable to create kubernetes client: %w", err)
}
common.LogInfo2Quiet("Waiting for node to exist")
nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
Clientset: clientset,
NodeName: nodeName,
RetryCount: 20,
})
if err != nil {
return fmt.Errorf("Error waiting for pod to exist: %w", err)
}
if len(nodes) == 0 {
return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled kubernetes.io/role=worker")
}
common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker")
err = clientset.LabelNode(ctx, LabelNodeInput{
Name: nodes[0].Name,
@@ -509,6 +513,16 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
}
}
common.LogInfo2Quiet("Annotating node with connection information")
err = clientset.AnnotateNode(ctx, AnnotateNodeInput{
Name: nodes[0].Name,
Key: "dokku.com/remote-host",
Value: remoteHost,
})
if err != nil {
return fmt.Errorf("Unable to patch node: %w", err)
}
common.LogVerboseQuiet("Done")
return nil
}

29
plugins/scheduler-k3s/triggers.go
View File

@@ -21,6 +21,7 @@ import (
"github.com/dokku/dokku/plugins/config"
"github.com/dokku/dokku/plugins/cron"
"github.com/fatih/color"
"github.com/hashicorp/go-multierror"
"github.com/kballard/go-shellquote"
"github.com/rancher/wharfie/pkg/registries"
"github.com/ryanuber/columnize"
@@ -121,7 +122,33 @@ func TriggerPostRegistryLogin(server string, username string) error {
}
// todo: auth against all nodes in cluster
return nil
clientset, err := NewKubernetesClient()
if err != nil {
return fmt.Errorf("Error creating kubernetes client: %w", err)
}
ctx := context.Background()
nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
if err != nil {
return fmt.Errorf("Error listing nodes: %w", err)
}
var result error
for _, node := range nodes {
remoteHost, ok := node.Annotations["dokku.com/remote-host"]
if !ok {
continue
}
err := copyRegistryToNode(ctx, remoteHost)
if err != nil {
wrappedErr := fmt.Errorf("Error copying registry to node: %w", err)
result = multierror.Append(result, wrappedErr)
common.LogWarn(wrappedErr.Error())
}
}
return result
}
// TriggerSchedulerDeploy deploys an image tag for a given application