fix: use image pull secrets instead of registries.yaml to reference private repositories

While this will now more or less require a local docker login - which can be overridden by setting the image-pull-secrets property - this ensures every deploy has a valid context.

One annoying thing is that the secret is per-app vs global, meaning that if the secret must be rotated, all apps need to be redeployed. A future change can fix that by deploying a global secret, or if folks really desire, this can be done by setting the image-pull-secrets property on the app/globally.

debian/control

@@ -3,7 +3,7 @@ Version: 0.33.5
 Section: web
 Priority: optional
 Architecture: amd64
-Depends: apache2-utils, acl, locales, git, cpio, curl, man-db, netcat, sshcommand, docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-buildx-plugin | moby-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util, python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip, util-linux
+Depends: apache2-utils, locales, git, cpio, curl, man-db, netcat, sshcommand, docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-buildx-plugin | moby-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util, python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip, util-linux
 Recommends: herokuish, bash-completion, dokku-update, dokku-event-listener
 Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn, sudo, python3, debconf
 Maintainer: Jose Diaz-Gonzalez <dokku@josediazgonzalez.com>

docs/deployment/schedulers/k3s.md

@@ -219,7 +219,7 @@ The default value may be set by passing an empty value for the option:
 dokku scheduler-k3s:set node-js-app image-pull-secrets
 ```
 
-The `image-pull-secrets` property can also be set globally. The global default is empty string, and k3s will use the local `registries.yaml` for any private registry pulls.
+The `image-pull-secrets` property can also be set globally. The global default is empty string, and k3s will use Dokku's locally configured `~/.docker/config.json` for any private registry pulls.
 
 ```shell
 dokku scheduler-k3s:set --global image-pull-secrets 60s

plugins/scheduler-k3s/Makefile

@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 

plugins/scheduler-k3s/functions.go

@@ -1120,10 +1120,6 @@ func isK3sInstalled() error {
 		return fmt.Errorf("k3s binary is not available")
 	}
 
-	if !common.FileExists(RegistryConfigPath) {
-		return fmt.Errorf("k3s registry config is not available")
-	}
-
 	if !common.FileExists(KubeConfigPath) {
 		return fmt.Errorf("k3s kubeconfig is not available")
 	}

plugins/scheduler-k3s/registry.go

@@ -1,78 +0,0 @@
-package scheduler_k3s
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/dokku/dokku/plugins/common"
-)
-
-func copyRegistryToNode(ctx context.Context, remoteHost string) error {
-	common.LogInfo1Quiet(fmt.Sprintf("Updating k3s registry configuration on %s", remoteHost))
-	sftpCmd, err := common.CallSftpCopy(common.SftpCopyInput{
-		AllowUknownHosts: true,
-		DestinationPath:  "/tmp/registries.yaml",
-		RemoteHost:       remoteHost,
-		SourcePath:       RegistryConfigPath,
-	})
-	if err != nil {
-		return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
-	}
-
-	if sftpCmd.ExitErr != nil {
-		return fmt.Errorf("Error copying registries.yaml to remote host: %s", sftpCmd.ExitErr.Error())
-	}
-
-	common.LogVerboseQuiet("Moving k3s registry configuration into place")
-	mvCmd, err := common.CallSshCommand(common.SshCommandInput{
-		AllowUknownHosts: true,
-		Args:             []string{"/tmp/registries.yaml", RegistryConfigPath},
-		Command:          "mv",
-		RemoteHost:       remoteHost,
-		StreamStdio:      true,
-		Sudo:             true,
-	})
-	if err != nil {
-		return fmt.Errorf("Unable to call mv command over ssh: %w", err)
-	}
-
-	if mvCmd.ExitCode != 0 {
-		return fmt.Errorf("Invalid exit code from mv command over ssh: %d", mvCmd.ExitCode)
-	}
-
-	common.LogVerboseQuiet("Updating k3s registry permissions")
-	chmodCmd, err := common.CallSshCommand(common.SshCommandInput{
-		AllowUknownHosts: true,
-		Args:             []string{"0644", RegistryConfigPath},
-		Command:          "chmod",
-		RemoteHost:       remoteHost,
-		StreamStdio:      true,
-		Sudo:             true,
-	})
-	if err != nil {
-		return fmt.Errorf("Unable to call chmod command over ssh: %w", err)
-	}
-
-	if chmodCmd.ExitCode != 0 {
-		return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
-	}
-
-	common.LogVerboseQuiet("Updating k3s registry ower")
-	chownCmd, err := common.CallSshCommand(common.SshCommandInput{
-		AllowUknownHosts: true,
-		Args:             []string{"root:root", RegistryConfigPath},
-		Command:          "chown",
-		RemoteHost:       remoteHost,
-		StreamStdio:      true,
-		Sudo:             true,
-	})
-	if err != nil {
-		return fmt.Errorf("Unable to call chown command over ssh: %w", err)
-	}
-
-	if chownCmd.ExitCode != 0 {
-		return fmt.Errorf("Invalid exit code from chown command over ssh: %d", chownCmd.ExitCode)
-	}
-
-	return nil
-}

plugins/scheduler-k3s/scheduler_k3s.go

@@ -42,7 +42,6 @@ var (
 const DefaultIngressClass = "traefik"
 const GlobalProcessType = "--global"
 const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
-const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml"
 
 var (
 	runtimeScheme  = runtime.NewScheme()

plugins/scheduler-k3s/src/triggers/triggers.go

@@ -35,10 +35,6 @@ func main() {
 	case "post-delete":
 		appName := flag.Arg(0)
 		err = scheduler_k3s.TriggerPostDelete(appName)
-	case "post-registry-login":
-		server := flag.Arg(0)
-		username := flag.Arg(1)
-		err = scheduler_k3s.TriggerPostRegistryLogin(server, username)
 	case "report":
 		appName := flag.Arg(0)
 		err = scheduler_k3s.ReportSingleApp(appName, "", "")

plugins/scheduler-k3s/subcommands.go

@@ -241,27 +241,6 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
 		return fmt.Errorf("Invalid exit code from k3s installer command: %d", installerCmd.ExitCode)
 	}
 
-	if err := common.TouchFile(RegistryConfigPath); err != nil {
-		return fmt.Errorf("Error creating initial registries.yaml file")
-	}
-
-	common.LogInfo2Quiet("Setting registries.yaml permissions")
-	registryAclCmd, err := common.CallExecCommand(common.ExecCommandInput{
-		Command: "setfacl",
-		Args: []string{
-			"-m",
-			"user:dokku:rwx",
-			RegistryConfigPath,
-		},
-		StreamStdio: true,
-	})
-	if err != nil {
-		return fmt.Errorf("Unable to call setfacl command: %w", err)
-	}
-	if registryAclCmd.ExitCode != 0 {
-		return fmt.Errorf("Invalid exit code from setfacl command: %d", registryAclCmd.ExitCode)
-	}
-
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
@@ -572,11 +551,6 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn
 		return fmt.Errorf("Unable to find node after joining cluster, node will not be annotated/labeled appropriately access registry secrets")
 	}
 
-	err = copyRegistryToNode(ctx, remoteHost)
-	if err != nil {
-		return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
-	}
-
 	labels := ServerLabels
 	if role == "worker" {
 		labels = WorkerLabels

plugins/scheduler-k3s/template.go

@@ -45,6 +45,7 @@ type GlobalValues struct {
 type GlobalImage struct {
 	ImagePullSecrets string `yaml:"image_pull_secrets"`
 	Name             string `yaml:"name"`
+	PullSecretBase64 string `yaml:"pull_secret_base64"`
 	Type             string `yaml:"type"`
 	WorkingDir       string `yaml:"working_dir"`
 }

plugins/scheduler-k3s/templates/chart/cron-job.yaml

@@ -111,7 +111,8 @@ spec:
             workingDir: {{ $.Values.global.image.working_dir }}
             {{- end }}
           {{- if $.Values.global.image.image_pull_secrets }}
-          imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }}
+          imagePullSecrets:
+          - name: {{ $.Values.global.image.image_pull_secrets }}
           {{- end }}
           restartPolicy: Never
           serviceAccountName: {{ $.Values.global.app_name }}

plugins/scheduler-k3s/templates/chart/deployment.yaml

@@ -125,6 +125,7 @@ spec:
         workingDir: {{ $.Values.global.image.working_dir }}
         {{- end }}
       {{- if $.Values.global.image.image_pull_secrets }}
-      imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }}
+      imagePullSecrets:
+      - name: {{ $.Values.global.image.image_pull_secrets }}
       {{- end }}
       serviceAccountName: {{ $.Values.global.app_name }}

plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml

@@ -0,0 +1,23 @@
+{{- if $.Values.global.image.pull_secret_base64 }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  annotations:
+    app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
+    dokku.com/managed: "true"
+    {{- if and $.Values.global.annotations $.Values.global.annotations.secret }}
+    {{- range $k, $v := $.Values.global.annotations.secret }}
+    {{ $k }}: {{ $v | quote }}
+    {{- end }}
+    {{- end }}
+  labels:
+    app.kubernetes.io/instance: image-pull-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
+    app.kubernetes.io/name: image-pull-{{ $.Values.global.app_name }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: image-pull-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
+  namespace: {{ $.Values.global.namespace }}
+data:
+  .dockerconfigjson: "{{ $.Values.global.image.pull_secret_base64 }}"
+type: kubernetes.io/dockerconfigjson
+{{- end }}

plugins/scheduler-k3s/templates/chart/secret.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Secret
 metadata:

plugins/scheduler-k3s/triggers.go

@@ -23,11 +23,8 @@ import (
 	"github.com/dokku/dokku/plugins/config"
 	"github.com/dokku/dokku/plugins/cron"
 	"github.com/fatih/color"
-	"github.com/hashicorp/go-multierror"
 	"github.com/kballard/go-shellquote"
-	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/ryanuber/columnize"
-	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/kubernetes/pkg/client/conditions"
@@ -78,88 +75,6 @@ func TriggerPostDelete(appName string) error {
 	return propertyErr
 }
 
-// TriggerPostRegistryLogin updates the `/etc/rancher/k3s/registries.yaml` to include
-// auth information for the registry. Note that if the file does not exist, it won't be updated.
-func TriggerPostRegistryLogin(server string, username string) error {
-	if !common.FileExists("/usr/local/bin/k3s") {
-		return nil
-	}
-
-	password := os.Getenv("DOCKER_REGISTRY_PASS")
-
-	yamlFile, err := os.ReadFile(RegistryConfigPath)
-	if err != nil {
-		return fmt.Errorf("Unable to read existing registries.yaml: %w", err)
-	}
-
-	var registry registries.Registry
-	err = yaml.Unmarshal(yamlFile, &registry)
-	if err != nil {
-		return fmt.Errorf("Unable to unmarshal registry configuration from yaml: %w", err)
-	}
-
-	common.LogInfo1("Updating k3s configuration")
-	if registry.Auths == nil {
-		registry.Auths = map[string]registries.AuthConfig{}
-	}
-
-	if server == "docker.io" {
-		server = "registry-1.docker.io"
-	}
-
-	registry.Auths[server] = registries.AuthConfig{
-		Username: username,
-		Password: password,
-	}
-
-	data, err := yaml.Marshal(&registry)
-	if err != nil {
-		return fmt.Errorf("Unable to marshal registry configuration to yaml: %w", err)
-	}
-
-	if err := os.WriteFile(RegistryConfigPath, data, os.FileMode(0644)); err != nil {
-		return fmt.Errorf("Unable to write registry configuration to file: %w", err)
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
-		syscall.SIGINT,
-		syscall.SIGQUIT,
-		syscall.SIGTERM)
-	go func() {
-		<-signals
-		cancel()
-	}()
-
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Error creating kubernetes client: %w", err)
-	}
-
-	nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
-	if err != nil {
-		return fmt.Errorf("Error listing nodes: %w", err)
-	}
-
-	var result error
-	for _, node := range nodes {
-		remoteHost, ok := node.Annotations["dokku.com/remote-host"]
-		if !ok {
-			continue
-		}
-
-		err := copyRegistryToNode(ctx, remoteHost)
-		if err != nil {
-			wrappedErr := fmt.Errorf("Error copying registry to node: %w", err)
-			result = multierror.Append(result, wrappedErr)
-			common.LogWarn(wrappedErr.Error())
-		}
-	}
-
-	return result
-}
-
 // TriggerSchedulerDeploy deploys an image tag for a given application
 func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) error {
 	if scheduler != "k3s" {
@@ -208,8 +123,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err)
 	}
 
-	imagePullSecrets := getComputedImagePullSecrets(appName)
-
 	imageSourceType := "dockerfile"
 	if common.IsImageCnbBased(image) {
 		imageSourceType = "pack"
@@ -256,7 +169,22 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	}
 
 	deploymentId := time.Now().Unix()
-	globalTemplateFiles := []string{"service-account", "secret"}
+	pullSecretBase64 := base64.StdEncoding.EncodeToString([]byte(""))
+	imagePullSecrets := getComputedImagePullSecrets(appName)
+	if imagePullSecrets == "" {
+		dockerConfigPath := filepath.Join(os.Getenv("DOKKU_ROOT"), ".docker/config.json")
+		if fi, err := os.Stat(dockerConfigPath); err == nil && !fi.IsDir() {
+			b, err := os.ReadFile(dockerConfigPath)
+			if err != nil {
+				return fmt.Errorf("Error reading docker config: %w", err)
+			}
+
+			imagePullSecrets = fmt.Sprintf("image-pull-%s.%d", appName, deploymentId)
+			pullSecretBase64 = base64.StdEncoding.EncodeToString(b)
+		}
+	}
+
+	globalTemplateFiles := []string{"service-account", "secret", "image-pull-secret"}
 	for _, templateName := range globalTemplateFiles {
 		b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
 		if err != nil {
@@ -345,6 +273,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			DeploymentID: fmt.Sprint(deploymentId),
 			Image: GlobalImage{
 				ImagePullSecrets: imagePullSecrets,
+				PullSecretBase64: pullSecretBase64,
 				Name:             image,
 				Type:             imageSourceType,
 				WorkingDir:       workingDir,