feat: copy the registry file to all remote hosts
This commit is contained in:
1 parent
a354210f67
commit
3375474a4c
5 files changed
+176
-26
No files matched your search
@@ -8,7 +8,9 @@ require (
|
|||||||
github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
|
github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
|
||||||
github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
|
github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
|
||||||
github.com/fatih/color v1.16.0
|
github.com/fatih/color v1.16.0
|
||||||
|
github.com/go-openapi/jsonpointer v0.19.6
|
||||||
github.com/go-resty/resty/v2 v2.11.0
|
github.com/go-resty/resty/v2 v2.11.0
|
||||||
|
github.com/hashicorp/go-multierror v1.1.1
|
||||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
|
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
|
||||||
github.com/rancher/wharfie v0.6.5
|
github.com/rancher/wharfie v0.6.5
|
||||||
github.com/ryanuber/columnize v2.1.2+incompatible
|
github.com/ryanuber/columnize v2.1.2+incompatible
|
||||||
@@ -66,7 +68,6 @@ require (
|
|||||||
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
|
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
|
||||||
github.com/go-logr/logr v1.3.0 // indirect
|
github.com/go-logr/logr v1.3.0 // indirect
|
||||||
github.com/go-logr/stdr v1.2.2 // indirect
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
github.com/go-openapi/jsonpointer v0.19.6 // indirect
|
|
||||||
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
||||||
github.com/go-openapi/swag v0.22.3 // indirect
|
github.com/go-openapi/swag v0.22.3 // indirect
|
||||||
github.com/gobwas/glob v0.2.3 // indirect
|
github.com/gobwas/glob v0.2.3 // indirect
|
||||||
@@ -84,7 +85,6 @@ require (
|
|||||||
github.com/gosuri/uitable v0.0.4 // indirect
|
github.com/gosuri/uitable v0.0.4 // indirect
|
||||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
|
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
|
||||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
|
||||||
github.com/huandu/xstrings v1.4.0 // indirect
|
github.com/huandu/xstrings v1.4.0 // indirect
|
||||||
github.com/imdario/mergo v0.3.13 // indirect
|
github.com/imdario/mergo v0.3.13 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
|
|||||||
@@ -73,6 +73,37 @@ func KubernetesClientConfig() clientcmd.ClientConfig {
|
|||||||
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
|
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
|
||||||
|
type AnnotateNodeInput struct {
|
||||||
|
// Name is the Kubernetes node name
|
||||||
|
Name string
|
||||||
|
// Key is the annotation key
|
||||||
|
Key string
|
||||||
|
// Value is the annotation value
|
||||||
|
Value string
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnnotateNode annotates a Kubernetes node
|
||||||
|
func (k KubernetesClient) AnnotateNode(ctx context.Context, input AnnotateNodeInput) error {
|
||||||
|
node, err := k.Client.CoreV1().Nodes().Get(ctx, input.Name, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if node == nil {
|
||||||
|
return errors.New("node is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
keyPath := fmt.Sprintf("/metadata/annotations/%s", jsonpointer.Escape(input.Key))
|
||||||
|
patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value)
|
||||||
|
_, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to annotate node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// CreateJobInput contains all the information needed to create a Kubernetes job
|
// CreateJobInput contains all the information needed to create a Kubernetes job
|
||||||
type CreateJobInput struct {
|
type CreateJobInput struct {
|
||||||
// Job is the Kubernetes job
|
// Job is the Kubernetes job
|
||||||
@@ -190,9 +221,9 @@ func (k KubernetesClient) LabelNode(ctx context.Context, input LabelNodeInput) e
|
|||||||
return errors.New("node is nil")
|
return errors.New("node is nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape("kubernetes.io/role"))
|
keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape(input.Key))
|
||||||
patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, "worker")
|
patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value)
|
||||||
_, err = k.Client.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
|
_, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to label node: %w", err)
|
return fmt.Errorf("failed to label node: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package scheduler_k3s
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/dokku/dokku/plugins/common"
|
||||||
|
)
|
||||||
|
|
||||||
|
func copyRegistryToNode(ctx context.Context, remoteHost string) error {
|
||||||
|
common.LogInfo1Quiet(fmt.Sprintf("Updating k3s registry configuration on %s", remoteHost))
|
||||||
|
sftpCmd, err := common.CallSftpCopy(common.SftpCopyInput{
|
||||||
|
AllowUknownHosts: true,
|
||||||
|
DestinationPath: "/tmp/registries.yaml",
|
||||||
|
RemoteHost: remoteHost,
|
||||||
|
SourcePath: RegistryConfigPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if sftpCmd.ExitErr != nil {
|
||||||
|
return fmt.Errorf("Error copying registries.yaml to remote host: %s", sftpCmd.ExitErr.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
common.LogVerboseQuiet("Moving k3s registry configuration into place")
|
||||||
|
mvCmd, err := common.CallSshCommand(common.SshCommandInput{
|
||||||
|
AllowUknownHosts: true,
|
||||||
|
Args: []string{"/tmp/registries.yaml", RegistryConfigPath},
|
||||||
|
Command: "mv",
|
||||||
|
RemoteHost: remoteHost,
|
||||||
|
StreamStdio: true,
|
||||||
|
Sudo: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to call mv command over ssh: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if mvCmd.ExitCode != 0 {
|
||||||
|
return fmt.Errorf("Invalid exit code from mv command over ssh: %d", mvCmd.ExitCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
common.LogVerboseQuiet("Updating k3s registry permissions")
|
||||||
|
chmodCmd, err := common.CallSshCommand(common.SshCommandInput{
|
||||||
|
AllowUknownHosts: true,
|
||||||
|
Args: []string{"0644", RegistryConfigPath},
|
||||||
|
Command: "chmod",
|
||||||
|
RemoteHost: remoteHost,
|
||||||
|
StreamStdio: true,
|
||||||
|
Sudo: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to call chmod command over ssh: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if chmodCmd.ExitCode != 0 {
|
||||||
|
return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
common.LogVerboseQuiet("Updating k3s registry ower")
|
||||||
|
chownCmd, err := common.CallSshCommand(common.SshCommandInput{
|
||||||
|
AllowUknownHosts: true,
|
||||||
|
Args: []string{"root:root", RegistryConfigPath},
|
||||||
|
Command: "chown",
|
||||||
|
RemoteHost: remoteHost,
|
||||||
|
StreamStdio: true,
|
||||||
|
Sudo: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to call chown command over ssh: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if chownCmd.ExitCode != 0 {
|
||||||
|
return fmt.Errorf("Invalid exit code from chown command over ssh: %d", chownCmd.ExitCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -462,7 +462,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
|
|||||||
args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
|
args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
|
||||||
}
|
}
|
||||||
|
|
||||||
common.LogInfo2Quiet("Joining k3s cluster")
|
common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
|
||||||
joinCmd, err := common.CallSshCommand(common.SshCommandInput{
|
joinCmd, err := common.CallSshCommand(common.SshCommandInput{
|
||||||
Command: "/tmp/k3s-installer.sh",
|
Command: "/tmp/k3s-installer.sh",
|
||||||
Args: args,
|
Args: args,
|
||||||
@@ -477,27 +477,31 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
|
|||||||
if joinCmd.ExitCode != 0 {
|
if joinCmd.ExitCode != 0 {
|
||||||
return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
|
return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
|
||||||
}
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
clientset, err := NewKubernetesClient()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to create kubernetes client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
common.LogInfo2Quiet("Waiting for node to exist")
|
||||||
|
nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
|
||||||
|
Clientset: clientset,
|
||||||
|
NodeName: nodeName,
|
||||||
|
RetryCount: 20,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error waiting for pod to exist: %w", err)
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled appropriately and cannot access registry secrets")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = copyRegistryToNode(ctx, remoteHost)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
if role == "worker" {
|
if role == "worker" {
|
||||||
ctx := context.Background()
|
|
||||||
clientset, err := NewKubernetesClient()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("Unable to create kubernetes client: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
common.LogInfo2Quiet("Waiting for node to exist")
|
|
||||||
nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
|
|
||||||
Clientset: clientset,
|
|
||||||
NodeName: nodeName,
|
|
||||||
RetryCount: 20,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("Error waiting for pod to exist: %w", err)
|
|
||||||
}
|
|
||||||
if len(nodes) == 0 {
|
|
||||||
return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled kubernetes.io/role=worker")
|
|
||||||
}
|
|
||||||
|
|
||||||
common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker")
|
common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker")
|
||||||
err = clientset.LabelNode(ctx, LabelNodeInput{
|
err = clientset.LabelNode(ctx, LabelNodeInput{
|
||||||
Name: nodes[0].Name,
|
Name: nodes[0].Name,
|
||||||
@@ -509,6 +513,16 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
common.LogInfo2Quiet("Annotating node with connection information")
|
||||||
|
err = clientset.AnnotateNode(ctx, AnnotateNodeInput{
|
||||||
|
Name: nodes[0].Name,
|
||||||
|
Key: "dokku.com/remote-host",
|
||||||
|
Value: remoteHost,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Unable to patch node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
common.LogVerboseQuiet("Done")
|
common.LogVerboseQuiet("Done")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import (
|
|||||||
"github.com/dokku/dokku/plugins/config"
|
"github.com/dokku/dokku/plugins/config"
|
||||||
"github.com/dokku/dokku/plugins/cron"
|
"github.com/dokku/dokku/plugins/cron"
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
|
"github.com/hashicorp/go-multierror"
|
||||||
"github.com/kballard/go-shellquote"
|
"github.com/kballard/go-shellquote"
|
||||||
"github.com/rancher/wharfie/pkg/registries"
|
"github.com/rancher/wharfie/pkg/registries"
|
||||||
"github.com/ryanuber/columnize"
|
"github.com/ryanuber/columnize"
|
||||||
@@ -121,7 +122,33 @@ func TriggerPostRegistryLogin(server string, username string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// todo: auth against all nodes in cluster
|
// todo: auth against all nodes in cluster
|
||||||
return nil
|
clientset, err := NewKubernetesClient()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error listing nodes: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var result error
|
||||||
|
for _, node := range nodes {
|
||||||
|
remoteHost, ok := node.Annotations["dokku.com/remote-host"]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err := copyRegistryToNode(ctx, remoteHost)
|
||||||
|
if err != nil {
|
||||||
|
wrappedErr := fmt.Errorf("Error copying registry to node: %w", err)
|
||||||
|
result = multierror.Append(result, wrappedErr)
|
||||||
|
common.LogWarn(wrappedErr.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// TriggerSchedulerDeploy deploys an image tag for a given application
|
// TriggerSchedulerDeploy deploys an image tag for a given application
|
||||||
|
|||||||
Reference in new issue
Block a user