From a80c73fb2c69077675517d910ed10789efeb26a5 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 19 Dec 2023 00:17:56 -0500
Subject: [PATCH 001/113] feat: create skeleton of k3s plugin

---
 docs/deployment/schedulers/k3s.md             | 194 ++++++++++++++++++
 plugins/scheduler-k3s/.gitignore              |   8 +
 plugins/scheduler-k3s/Makefile                |   6 +
 plugins/scheduler-k3s/go.mod                  |  20 ++
 plugins/scheduler-k3s/plugin.toml             |   4 +
 plugins/scheduler-k3s/report.go               |  24 +++
 plugins/scheduler-k3s/scheduler_k3s.go        |  12 ++
 .../scheduler-k3s/src/commands/commands.go    |  56 +++++
 .../src/subcommands/subcommands.go            |  50 +++++
 .../scheduler-k3s/src/triggers/triggers.go    |  45 ++++
 plugins/scheduler-k3s/subcommands.go          |  29 +++
 plugins/scheduler-k3s/triggers.go             |  51 +++++
 12 files changed, 499 insertions(+)
 create mode 100644 docs/deployment/schedulers/k3s.md
 create mode 100644 plugins/scheduler-k3s/.gitignore
 create mode 100644 plugins/scheduler-k3s/Makefile
 create mode 100644 plugins/scheduler-k3s/go.mod
 create mode 100644 plugins/scheduler-k3s/plugin.toml
 create mode 100644 plugins/scheduler-k3s/report.go
 create mode 100644 plugins/scheduler-k3s/scheduler_k3s.go
 create mode 100644 plugins/scheduler-k3s/src/commands/commands.go
 create mode 100644 plugins/scheduler-k3s/src/subcommands/subcommands.go
 create mode 100644 plugins/scheduler-k3s/src/triggers/triggers.go
 create mode 100644 plugins/scheduler-k3s/subcommands.go
 create mode 100644 plugins/scheduler-k3s/triggers.go

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
new file mode 100644
index 000000000..4c96b9053
--- /dev/null
+++ b/docs/deployment/schedulers/k3s.md
@@ -0,0 +1,194 @@
+# K3s Scheduler
+
+```
+scheduler-k3s:initialize [<--local>|<--remote user:password@host:port>] # initializes a cluster
+scheduler-k3s:add-server [user:password@host:port]                      # adds a server node to a Dokku-managed cluster
+scheduler-k3s:add-client [user:password@host:port]                      # adds a client node to a Dokku-managed cluster
+scheduler-k3s:set [<app>|--global] <key> (<value>)                      # Set or clear a scheduler-k3s property for an app or the scheduler
+```
+
+> [!IMPORTANT]
+> New as of 0.33.0
+
+For multi-server support, Dokku provides the ability for users to setup a K3s cluster. As with all schedulers, it is set on a per-app basis. The scheduler can currently be overridden by running the following command:
+
+```shell
+dokku scheduler:set node-js-app selected k3s
+```
+
+As it is the default, unsetting the `selected` scheduler property is also a valid way to reset the scheduler.
+
+```shell
+dokku scheduler:set node-js-app k3s
+```
+
+## Tutorial
+
+### Single-node cluster
+
+Initialize the cluster in single-node mode. This will start k3s on the Dokku node itself.
+
+```shell
+dokku scheduler-k3s:initialize
+```
+
+The above command will initialize a cluster with an automatically generated token for authentication. This token should be stored securely for later recovery, and can be displayed with via the `scheduler-k3s:report` command:
+
+```shell
+dokku scheduler-k3s:report --global
+```
+
+```
+=====> --global scheduler-k3s information
+       scheduler k3s token: topsecret:server:token
+```
+
+Setup registry authentication. The K3s scheduler plugin will automatically pick up any newly configured registry backends and ensure nodes in the cluster have the credentials in place for pulling images from the cluster.
+
+```shell
+# hub.docker.com
+dokku registry:login hub.docker.com $USERNAME $PASSWORD
+```
+
+To ensure images are pushed and pulled from the correct registry, set the correct `server` registry property. This can be set on a per-app basis, but we will set it globally here for this tutorial.
+
+```shell
+dokku registry:set --global server hub.docker.com
+```
+
+If using docker hub, you'll need to use a custom repository name. As it is unique, this must be set on a per-app basis.
+
+```shell
+dokku registry:set node-js-app image-repo my-awesome-prefix/node-js-app
+```
+
+Additionally, apps should be configured to push images on the release phase via the `push-on-release` registry property.
+
+```shell
+dokku registry:set --global push-on-release true
+```
+
+As routing is handled by traefik managed on the k3s plugin, set the proxy plugin to `k3s` as well. This implements a traefik-plugin compatible layer, allowing users to otherwise interact with the `traefik` plugin as expected.
+
+```shell
+dokku proxy:set --global k3s
+```
+
+Finally, set the scheduler to `k3s` so that app deploys will work on k3s.
+
+```shell
+dokku scheduler:set --global selected k3s
+```
+
+At this point, all app deploys will be performed against the k3s cluster.
+
+### Running a multi-cluster node
+
+For high-availability, it is recommended to add both agent and server nodes to the cluster. Dokku will default to starting the cluster with an embedded Etcd database backend, and is ready to add new agent or server nodes immediately.
+
+Multiple server nodes can be added with the `scheduler-k3s:add-server` command. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in server mode.
+
+> [!WARNING]
+> Certain ports must be open for cross-server communication. Refer to the [K3s networking documentation](https://docs.k3s.io/installation/requirements?os=debian#networking) for the required open ports between servers prior to running the command.
+
+```shell
+dokku scheduler-k3s:add-server root@server-1.example.com
+```
+
+Server nodes are typically used to replicate the cluster state, and it is recommended to have an odd number of nodes spread across several availability zones (datacenters in close proximity within a region). This allows for higher availability in the event of a cluster failure. Server nodes do not run any app workloads, but run control-plane services such as the traefik load balancer and the etcd backing store.
+
+> [!NOTE]
+> Only the initial Dokku server will be properly configured for deployment, and should be considered your git remote. Additional server nodes are for ensuring high-availability of the K3s etcd state. Ensure this server is properly backed up and restorable or deployments will not work.
+
+Agent nodes are used to run. To add an agent, run the `add-agent` command. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in agent mode. Agents are typically used to run app workloads.
+
+```shell
+dokku scheduler-k3s:add-agent root@server-1.example.com
+```
+
+When attaching an agent or server node, the K3s plugin will look at the IP associated with the `eth0` interface and use that to connect the new node to the cluster. To change this, set the `network-interface` property to the appropriate value.
+
+```shell
+dokku scheduler-k3s:set --global network-interface eth1
+```
+
+Dokku does not manage the ssh key of the server, but the value that should be added to the remote root user's `/root/.ssh/authorized_keys` file can be checked with the `git:public-key` command:
+
+```shell
+dokku git:public-key
+```
+
+### Joining an existing cluster
+
+> [!IMPORTANT]
+> This is future planned behavior. Commands here will not work as expected.
+
+In some cases, Dokku must be setup from scratch and joined to an existing cluster. To do so, first set the k3s token value:
+
+```shell
+dokku scheduler-k3s:set --global token topsecret:server:token
+```
+
+Next, run the `scheduler-k3s:join-cluster` command. This command takes a single server node in the cluster, will SSH onto the specified server, retrieve the kubeconfig, and then configure kubectl on the Dokku node to speak with the specified k3s cluster. Dokku will also periodically retrieve all server nodes in the cluster so that server nodes can be safely replaced as needed.
+
+```shell
+dokku scheduler-k3s:join-cluster root@server-1.example.com
+```
+
+### Using kubectl remotely
+
+> [!WARNING]
+> Certain ports must be open for interacting with the remote kubernets api. Refer to the [K3s networking documentation](https://docs.k3s.io/installation/requirements?os=debian#networking) for the required open ports between servers prior to running the command.
+
+By default, Dokku assumes that all it controls all actions on the cluster, and thus does not expose the `kubectl` binary for administrators. To interact with kubectl, you will need to retrieve the `kubeconfig` for the cluster and configure your client to use that configuration.
+
+```shell
+dokku scheduler-k3s:show-kubeconfig
+```
+
+## Scheduler Interface
+
+The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler.
+
+### Implemented Commands and Triggers
+
+This plugin implements various functionality through `plugn` triggers to integrate with Docker for running apps on a single server. The following functionality is supported by the `scheduler-docker-local` plugin.
+
+- `apps:destroy`
+- `deploy`
+- `logs`
+- `ps:stop`
+
+### Unimplemented command functionality
+
+- `apps:clone`
+- `apps:rename`
+- `ps:inspect`
+- `enter`
+- `run`
+- `run:detached`
+
+The following Dokku functionality is not implemented at this time.
+
+- `cron`
+- `vector` log integration
+- one-off tasks
+- persistent storage
+
+### Logging support
+
+App logs for the `logs` command are fetched by Dokku from running containers via the `kubectl` cli. Persisting logs via Vector is not implemented at this time.
+
+### Supported Resource Management Properties
+
+The `docker-local` scheduler supports a minimal list of resource _limits_ and _reservations_. The following properties are supported:
+
+#### Resource Limits
+
+- cpu: is specified in number of CPUs a process can access.
+- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
+
+#### Resource Reservations
+
+- cpu: is specified in number of CPUs a process can access.
+- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
diff --git a/plugins/scheduler-k3s/.gitignore b/plugins/scheduler-k3s/.gitignore
new file mode 100644
index 000000000..c6bcef78b
--- /dev/null
+++ b/plugins/scheduler-k3s/.gitignore
@@ -0,0 +1,8 @@
+/commands
+/subcommands/*
+/triggers/*
+/triggers
+/scheduler-*
+/install
+/post-*
+/report
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
new file mode 100644
index 000000000..fc07a525d
--- /dev/null
+++ b/plugins/scheduler-k3s/Makefile
@@ -0,0 +1,6 @@
+SUBCOMMANDS = subcommands/report subcommands/set
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report
+BUILD = commands subcommands triggers
+PLUGIN_NAME = scheduler
+
+include ../../common.mk
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
new file mode 100644
index 000000000..2d57fe982
--- /dev/null
+++ b/plugins/scheduler-k3s/go.mod
@@ -0,0 +1,20 @@
+module github.com/dokku/dokku/plugins/scheduler
+
+go 1.21
+
+require (
+	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
+	github.com/spf13/pflag v1.0.5
+)
+
+require (
+	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
+	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
+	github.com/joho/godotenv v1.2.0 // indirect
+	github.com/otiai10/copy v1.14.0 // indirect
+	github.com/ryanuber/columnize v2.1.2+incompatible // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+)
+
+replace github.com/dokku/dokku/plugins/common => ../common
diff --git a/plugins/scheduler-k3s/plugin.toml b/plugins/scheduler-k3s/plugin.toml
new file mode 100644
index 000000000..a4bf65e20
--- /dev/null
+++ b/plugins/scheduler-k3s/plugin.toml
@@ -0,0 +1,4 @@
+[plugin]
+description = "dokku core scheduler-k3s plugin"
+version = "0.32.3"
+[plugin.config]
diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go
new file mode 100644
index 000000000..b5a7672fe
--- /dev/null
+++ b/plugins/scheduler-k3s/report.go
@@ -0,0 +1,24 @@
+package scheduler_k3s
+
+import (
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// ReportSingleApp is an internal function that displays the scheduler-k3s report for one or more apps
+func ReportSingleApp(appName string, format string, infoFlag string) error {
+	if err := common.VerifyAppName(appName); err != nil {
+		return err
+	}
+
+	flags := map[string]common.ReportFunc{}
+
+	flagKeys := []string{}
+	for flagKey := range flags {
+		flagKeys = append(flagKeys, flagKey)
+	}
+
+	trimPrefix := false
+	uppercaseFirstCharacter := true
+	infoFlags := common.CollectReport(appName, infoFlag, flags)
+	return common.ReportSingleApp("scheduler-k3s", appName, infoFlag, infoFlags, flagKeys, format, trimPrefix, uppercaseFirstCharacter)
+}
diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
new file mode 100644
index 000000000..88b5df958
--- /dev/null
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -0,0 +1,12 @@
+package scheduler_k3s
+
+var (
+	// DefaultProperties is a map of all valid network properties with corresponding default property values
+	DefaultProperties = map[string]string{}
+
+	// GlobalProperties is a map of all valid global network properties
+	GlobalProperties = map[string]bool{
+		"network-interface": true,
+		"token":             true,
+	}
+)
diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go
new file mode 100644
index 000000000..d36ffaf49
--- /dev/null
+++ b/plugins/scheduler-k3s/src/commands/commands.go
@@ -0,0 +1,56 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+const (
+	helpHeader = `Usage: dokku scheduler-k3s[:COMMAND]
+
+Manage scheduler-k3s settings for an app
+
+Additional commands:`
+
+	helpContent = `
+    scheduler-k3s:report [<app>] [<flag>], Displays a scheduler-k3s report for one or more apps
+    scheduler-k3s:set <app> <property> (<value>), Set or clear a scheduler-k3s property for an app
+`
+)
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+
+	cmd := flag.Arg(0)
+	switch cmd {
+	case "scheduler-k3s", "scheduler-k3s:help":
+		usage()
+	case "help":
+		command := common.NewShellCmd(fmt.Sprintf("ps -o command= %d", os.Getppid()))
+		command.ShowOutput = false
+		output, err := command.Output()
+
+		if err == nil && strings.Contains(string(output), "--all") {
+			fmt.Println(helpContent)
+		} else {
+			fmt.Print("\n    scheduler-k3s, Manage scheduler-k3s settings for an app\n")
+		}
+	default:
+		dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT"))
+		if err != nil {
+			fmt.Println("failed to retrieve DOKKU_NOT_IMPLEMENTED_EXIT environment variable")
+			dokkuNotImplementExitCode = 10
+		}
+		os.Exit(dokkuNotImplementExitCode)
+	}
+}
+
+func usage() {
+	common.CommandUsage(helpHeader, helpContent)
+}
diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go
new file mode 100644
index 000000000..ea2ac1b1f
--- /dev/null
+++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go
@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
+	scheduler_k3s "github.com/dokku/dokku/plugins/scheduler-k3s"
+
+	flag "github.com/spf13/pflag"
+)
+
+// main entrypoint to all subcommands
+func main() {
+	parts := strings.Split(os.Args[0], "/")
+	subcommand := parts[len(parts)-1]
+
+	var err error
+	switch subcommand {
+	case "report":
+		args := flag.NewFlagSet("scheduler-k3s:report", flag.ExitOnError)
+		format := args.String("format", "stdout", "format: [ stdout | json ]")
+		osArgs, infoFlag, flagErr := common.ParseReportArgs("scheduler-k3s", os.Args[2:])
+		if flagErr == nil {
+			args.Parse(osArgs)
+			appName := args.Arg(0)
+			err = scheduler_k3s.CommandReport(appName, *format, infoFlag)
+		}
+	case "set":
+		args := flag.NewFlagSet("scheduler-k3s:set", flag.ExitOnError)
+		global := args.Bool("global", false, "--global: set a global property")
+		args.Parse(os.Args[2:])
+		appName := args.Arg(0)
+		property := args.Arg(1)
+		value := args.Arg(2)
+		if *global {
+			appName = "--global"
+			property = args.Arg(0)
+			value = args.Arg(1)
+		}
+		err = scheduler_k3s.CommandSet(appName, property, value)
+	default:
+		err = fmt.Errorf("Invalid plugin subcommand call: %s", subcommand)
+	}
+
+	if err != nil {
+		common.LogFailWithError(err)
+	}
+}
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
new file mode 100644
index 000000000..24c9e01cf
--- /dev/null
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	flag "github.com/spf13/pflag"
+
+	"github.com/dokku/dokku/plugins/common"
+	scheduler_k3s "github.com/dokku/dokku/plugins/scheduler-k3s"
+)
+
+// main entrypoint to all triggers
+func main() {
+	parts := strings.Split(os.Args[0], "/")
+	trigger := parts[len(parts)-1]
+	flag.Parse()
+
+	var err error
+	switch trigger {
+	case "install":
+		err = scheduler_k3s.TriggerInstall()
+	case "post-app-clone-setup":
+		oldAppName := flag.Arg(0)
+		newAppName := flag.Arg(1)
+		err = scheduler_k3s.TriggerPostAppCloneSetup(oldAppName, newAppName)
+	case "post-app-rename-setup":
+		oldAppName := flag.Arg(0)
+		newAppName := flag.Arg(1)
+		err = scheduler_k3s.TriggerPostAppRenameSetup(oldAppName, newAppName)
+	case "post-delete":
+		appName := flag.Arg(0)
+		err = scheduler_k3s.TriggerPostDelete(appName)
+	case "report":
+		appName := flag.Arg(0)
+		err = scheduler_k3s.ReportSingleApp(appName, "", "")
+	default:
+		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
+	}
+
+	if err != nil {
+		common.LogFailWithError(err)
+	}
+}
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
new file mode 100644
index 000000000..18dcb61db
--- /dev/null
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -0,0 +1,29 @@
+package scheduler_k3s
+
+import (
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// CommandReport displays a scheduler-k3s report for one or more apps
+func CommandReport(appName string, format string, infoFlag string) error {
+	if len(appName) == 0 {
+		apps, err := common.DokkuApps()
+		if err != nil {
+			return err
+		}
+		for _, appName := range apps {
+			if err := ReportSingleApp(appName, format, infoFlag); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	return ReportSingleApp(appName, format, infoFlag)
+}
+
+// CommandSet set or clear a scheduler-k3s property for an app
+func CommandSet(appName string, property string, value string) error {
+	common.CommandPropertySet("scheduler-k3s", appName, property, value, DefaultProperties, GlobalProperties)
+	return nil
+}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
new file mode 100644
index 000000000..016d7d01a
--- /dev/null
+++ b/plugins/scheduler-k3s/triggers.go
@@ -0,0 +1,51 @@
+package scheduler_k3s
+
+import (
+	"fmt"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// TriggerInstall runs the install step for the scheduler-k3s plugin
+func TriggerInstall() error {
+	if err := common.PropertySetup("scheduler-k3s"); err != nil {
+		return fmt.Errorf("Unable to install the scheduler-k3s plugin: %s", err.Error())
+	}
+
+	return nil
+}
+
+// TriggerPostAppCloneSetup creates new scheduler-k3s files
+func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
+	err := common.PropertyClone("scheduler-k3s", oldAppName, newAppName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// TriggerPostAppRenameSetup renames scheduler-k3s files
+func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
+	if err := common.PropertyClone("scheduler-k3s", oldAppName, newAppName); err != nil {
+		return err
+	}
+
+	if err := common.PropertyDestroy("scheduler-k3s", oldAppName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// TriggerPostDelete destroys the scheduler-k3s data for a given app container
+func TriggerPostDelete(appName string) error {
+	dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName)
+	propertyErr := common.PropertyDestroy("scheduler-k3s", appName)
+
+	if dataErr != nil {
+		return dataErr
+	}
+
+	return propertyErr
+}

From 3821d22b93b010b7fba092f951954e9c36edaf1c Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 19 Dec 2023 00:36:03 -0500
Subject: [PATCH 002/113] chore: write go.sum file

---
 plugins/scheduler-k3s/go.sum | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 plugins/scheduler-k3s/go.sum

diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
new file mode 100644
index 000000000..bed9cc52d
--- /dev/null
+++ b/plugins/scheduler-k3s/go.sum
@@ -0,0 +1,8 @@
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
+github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
+github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
+github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

From 523d7bcf9e298456d40e8fa08138c88b29e1184b Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 12 Jan 2024 01:07:07 -0500
Subject: [PATCH 003/113] chore: bump golang.org/x/sync from 0.5.0 to 0.6.0

---
 plugins/scheduler-k3s/go.mod |  5 ++---
 plugins/scheduler-k3s/go.sum | 22 ++++++++++++++++++++--
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 2d57fe982..22ba5aafe 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -1,4 +1,4 @@
-module github.com/dokku/dokku/plugins/scheduler
+module github.com/dokku/dokku/plugins/scheduler-k3s
 
 go 1.21
 
@@ -10,10 +10,9 @@ require (
 require (
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
 	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
-	github.com/joho/godotenv v1.2.0 // indirect
 	github.com/otiai10/copy v1.14.0 // indirect
 	github.com/ryanuber/columnize v2.1.2+incompatible // indirect
-	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 )
 
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index bed9cc52d..d7dea1aac 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -1,8 +1,26 @@
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
+github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
-github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
+github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
+github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF25UfsK2X7Ph/zCAk=
 github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

From d6824c44c7ee767360b85f32ad23ffe9057041cc Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 12 Jan 2024 01:12:42 -0500
Subject: [PATCH 004/113] fix: correct plugin name

---
 plugins/scheduler-k3s/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index fc07a525d..c693d1e2f 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,6 +1,6 @@
 SUBCOMMANDS = subcommands/report subcommands/set
 TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report
 BUILD = commands subcommands triggers
-PLUGIN_NAME = scheduler
+PLUGIN_NAME = scheduler-k3s
 
 include ../../common.mk

From ec538ce1c53d7569f914b9f7ec1cb3e577459418 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 12 Jan 2024 02:25:22 -0500
Subject: [PATCH 005/113] feat: implement scheduler-k3s:initialize

---
 docs/deployment/schedulers/k3s.md             |  9 ++-
 dokku                                         |  4 +-
 plugins/scheduler-k3s/Makefile                |  2 +-
 plugins/scheduler-k3s/go.mod                  |  2 +
 plugins/scheduler-k3s/go.sum                  | 40 ++++++++++
 .../scheduler-k3s/src/commands/commands.go    |  1 +
 .../src/subcommands/subcommands.go            |  4 +
 plugins/scheduler-k3s/subcommands.go          | 74 +++++++++++++++++++
 8 files changed, 129 insertions(+), 7 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 4c96b9053..bfb87feca 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -1,10 +1,10 @@
 # K3s Scheduler
 
 ```
-scheduler-k3s:initialize [<--local>|<--remote user:password@host:port>] # initializes a cluster
-scheduler-k3s:add-server [user:password@host:port]                      # adds a server node to a Dokku-managed cluster
-scheduler-k3s:add-client [user:password@host:port]                      # adds a client node to a Dokku-managed cluster
-scheduler-k3s:set [<app>|--global] <key> (<value>)                      # Set or clear a scheduler-k3s property for an app or the scheduler
+scheduler-k3s:initialize                           # initializes a cluster
+scheduler-k3s:add-server [user:password@host:port] # adds a server node to a Dokku-managed cluster
+scheduler-k3s:add-client [user:password@host:port] # adds a client node to a Dokku-managed cluster
+scheduler-k3s:set [<app>|--global] <key> (<value>) # Set or clear a scheduler-k3s property for an app or the scheduler
 ```
 
 > [!IMPORTANT]
@@ -29,6 +29,7 @@ dokku scheduler:set node-js-app k3s
 Initialize the cluster in single-node mode. This will start k3s on the Dokku node itself.
 
 ```shell
+# must be run as root
 dokku scheduler-k3s:initialize
 ```
 
diff --git a/dokku b/dokku
index fdef028b2..ba7355a05 100755
--- a/dokku
+++ b/dokku
@@ -89,14 +89,14 @@ fi
 
 if [[ $(id -un) != "dokku" ]]; then
   unset TMP TMPDIR TEMP TEMPDIR
-  if [[ ! $1 =~ plugin:* ]] && [[ $1 != "ssh-keys:add" ]] && [[ $1 != "ssh-keys:remove" ]]; then
+  if [[ ! $1 =~ plugin:* ]] && [[ $1 != "ssh-keys:add" ]] && [[ $1 != "ssh-keys:remove" ]] && [[ $1 != "scheduler-k3s:initialize" ]]; then
     export SSH_USER=$(id -un)
     sudo -u dokku -E -H "$0" "$@"
     exit $?
   fi
 fi
 
-if [[ $1 =~ ^plugin:.* && $1 != "plugin:help" && $1 != "plugin:list" ]] || [[ $1 == "ssh-keys:add" ]] || [[ $1 == "ssh-keys:remove" ]]; then
+if [[ $1 =~ ^plugin:.* && $1 != "plugin:help" && $1 != "plugin:list" ]] || [[ $1 == "ssh-keys:add" ]] || [[ $1 == "ssh-keys:remove" ]] || [[ $1 == "scheduler-k3s:initialize" ]]; then
   if [[ $(id -un) != "root" ]]; then
     dokku_log_fail "This command must be run as root"
   else
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index c693d1e2f..40bdea3e5 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/report subcommands/set
+SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set
 TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 22ba5aafe..e5b9086b8 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
+	github.com/go-resty/resty/v2 v2.11.0
 	github.com/spf13/pflag v1.0.5
 )
 
@@ -12,6 +13,7 @@ require (
 	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
 	github.com/otiai10/copy v1.14.0 // indirect
 	github.com/ryanuber/columnize v2.1.2+incompatible // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 )
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index d7dea1aac..583dd0529 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -2,6 +2,8 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
+github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
+github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
@@ -14,13 +16,51 @@ github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF
 github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go
index d36ffaf49..c6f07ab76 100644
--- a/plugins/scheduler-k3s/src/commands/commands.go
+++ b/plugins/scheduler-k3s/src/commands/commands.go
@@ -18,6 +18,7 @@ Manage scheduler-k3s settings for an app
 Additional commands:`
 
 	helpContent = `
+    scheduler-k3s:initialize, Initializes a cluster
     scheduler-k3s:report [<app>] [<flag>], Displays a scheduler-k3s report for one or more apps
     scheduler-k3s:set <app> <property> (<value>), Set or clear a scheduler-k3s property for an app
 `
diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go
index ea2ac1b1f..bc6bd03ab 100644
--- a/plugins/scheduler-k3s/src/subcommands/subcommands.go
+++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go
@@ -18,6 +18,10 @@ func main() {
 
 	var err error
 	switch subcommand {
+	case "initialize":
+		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		err = scheduler_k3s.CommandInitialize()
 	case "report":
 		args := flag.NewFlagSet("scheduler-k3s:report", flag.ExitOnError)
 		format := args.String("format", "stdout", "format: [ stdout | json ]")
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 18dcb61db..cf3dc6b8e 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -1,9 +1,83 @@
 package scheduler_k3s
 
 import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
 	"github.com/dokku/dokku/plugins/common"
+	resty "github.com/go-resty/resty/v2"
 )
 
+// CommandInitialize initializes a k3s cluster on the local server
+func CommandInitialize() error {
+	client := resty.New()
+	resp, err := client.R().
+		Get("https://get.k3s.io")
+	if err != nil {
+		return fmt.Errorf("Unable to download k3s installer: %w", err)
+	}
+	if resp == nil {
+		return fmt.Errorf("Missing response from k3s installer download: %w", err)
+	}
+
+	if resp.StatusCode() != 200 {
+		return fmt.Errorf("Invalid status code for k3s installer script: %d", resp.StatusCode())
+	}
+
+	f, err := os.CreateTemp("", "sample")
+	if err != nil {
+		return fmt.Errorf("Unable to create temporary file for k3s installer: %w", err)
+	}
+	defer os.Remove(f.Name())
+
+	err = common.WriteSliceToFile(common.WriteSliceToFileInput{
+		Filename: f.Name(),
+		Lines:    strings.Split(resp.String(), "\n"),
+		Mode:     os.FileMode(755),
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to write k3s installer to file: %w", err)
+	}
+
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("Unable to close k3s installer file: %w", err)
+	}
+
+	fi, err := os.Stat(f.Name())
+	if err != nil {
+		return fmt.Errorf("Unable to get k3s installer file size: %w", err)
+	}
+
+	if fi.Size() == 0 {
+		return fmt.Errorf("Invalid k3s installer filesize")
+	}
+
+	time.Sleep(1 * time.Second)
+
+	// todo: allow this to be passed as an option or environment variable
+	token := "password"
+	if err := CommandSet("--global", "token", token); err != nil {
+		return fmt.Errorf("Unable to set k3s token: %w", err)
+	}
+
+	installerCmd := common.NewShellCmd(strings.Join([]string{
+		f.Name(),
+		// initialize the cluster
+		"--cluster-init",
+		// allow access for the dokku user
+		"--write-kubeconfig-mode", "0644",
+		// specify a token
+		"--token", token,
+	}, " "))
+	if !installerCmd.Execute() {
+		return fmt.Errorf("Error installing k3s: %w", installerCmd.ExitError)
+	}
+
+	return nil
+}
+
 // CommandReport displays a scheduler-k3s report for one or more apps
 func CommandReport(appName string, format string, infoFlag string) error {
 	if len(appName) == 0 {

From 65d840e4bc179f40aada7f07a860385db138086e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 12 Jan 2024 02:39:51 -0500
Subject: [PATCH 006/113] chore: stub out post-registry-login

---
 plugins/scheduler-k3s/Makefile                 | 2 +-
 plugins/scheduler-k3s/src/triggers/triggers.go | 5 +++++
 plugins/scheduler-k3s/triggers.go              | 8 ++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 40bdea3e5..3f64fb883 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index 24c9e01cf..1d9ab3b90 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -32,6 +32,11 @@ func main() {
 	case "post-delete":
 		appName := flag.Arg(0)
 		err = scheduler_k3s.TriggerPostDelete(appName)
+	case "post-registry-login":
+		server := flag.Arg(0)
+		username := flag.Arg(1)
+		password := flag.Arg(2)
+		err = scheduler_k3s.TriggerPostRegistryLogin(server, username, password)
 	case "report":
 		appName := flag.Arg(0)
 		err = scheduler_k3s.ReportSingleApp(appName, "", "")
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 016d7d01a..95ba710dc 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -49,3 +49,11 @@ func TriggerPostDelete(appName string) error {
 
 	return propertyErr
 }
+
+// TriggerPostRegistryLogin updates the `/etc/rancher/k3s/registries.yaml` to include
+// auth information for the registry
+func TriggerPostRegistryLogin(server string, username string, password string) error {
+	// todo: setup auth
+	// todo: auth against all nodes in cluster
+	return nil
+}

From f377c23403a6e80d33be3a85503702b0ed556b94 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 12 Jan 2024 23:55:42 -0500
Subject: [PATCH 007/113] feat: setup k3s registry automatically on registry
 login

---
 debian/control                       |  2 +-
 plugins/scheduler-k3s/go.mod         | 17 ++++++
 plugins/scheduler-k3s/go.sum         | 82 ++++++++++++++++++++++++++++
 plugins/scheduler-k3s/subcommands.go | 15 ++++-
 plugins/scheduler-k3s/triggers.go    | 45 ++++++++++++++-
 5 files changed, 157 insertions(+), 4 deletions(-)

diff --git a/debian/control b/debian/control
index 6e626b065..5fc0fc9b9 100644
--- a/debian/control
+++ b/debian/control
@@ -3,7 +3,7 @@ Version: 0.32.4
 Section: web
 Priority: optional
 Architecture: amd64
-Depends: apache2-utils, locales, git, cpio, curl, man-db, netcat, sshcommand, docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-buildx-plugin | moby-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util, python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip, util-linux
+Depends: apache2-utils, acl, locales, git, cpio, curl, man-db, netcat, sshcommand, docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-buildx-plugin | moby-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util, python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip, util-linux
 Recommends: herokuish, bash-completion, dokku-update, dokku-event-listener
 Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn, sudo, python3, debconf
 Maintainer: Jose Diaz-Gonzalez <dokku@josediazgonzalez.com>
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index e5b9086b8..6b9d42898 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -5,17 +5,34 @@ go 1.21
 require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
 	github.com/go-resty/resty/v2 v2.11.0
+	github.com/rancher/wharfie v0.6.4
 	github.com/spf13/pflag v1.0.5
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
 	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
+	github.com/docker/cli v20.10.20+incompatible // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/docker/docker v20.10.26+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/google/go-containerregistry v0.12.2-0.20230106184643-b063f6aeac72 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
 	github.com/otiai10/copy v1.14.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/ryanuber/columnize v2.1.2+incompatible // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 replace github.com/dokku/dokku/plugins/common => ../common
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index 583dd0529..4fa3487e9 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -2,21 +2,84 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
+github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0=
+github.com/containerd/stargz-snapshotter/estargz v0.12.1/go.mod h1:12VUuCq3qPq4y8yUW+l5w3+oXV3cx2Po3KSe/SmPGqw=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/cli v20.10.20+incompatible h1:lWQbHSHUFs7KraSN2jOJK7zbMS2jNCHI4mt4xUFUVQ4=
+github.com/docker/cli v20.10.20+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
+github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v20.10.26+incompatible h1:yhVdEnGdYdwRo2RAGciZU3P55Y7XH/ybb5A3L1d1rDE=
+github.com/docker/docker v20.10.26+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
+github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
 github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-containerregistry v0.12.2-0.20230106184643-b063f6aeac72 h1:mjNVVKGHs3+xtaGPnTHz4ozBphZNsxsgqFR4+TNwRVM=
+github.com/google/go-containerregistry v0.12.2-0.20230106184643-b063f6aeac72/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
+github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
+github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
 github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
 github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S+WV0JA6A=
+github.com/rancher/dynamiclistener v0.3.5/go.mod h1:dW/YF6/m2+uEyJ5VtEcd9THxda599HP6N9dSXk81+k0=
+github.com/rancher/wharfie v0.6.4 h1:JwYB+q661n8ut/ysgsjKe0P0z6bHCCFoC+29995ME90=
+github.com/rancher/wharfie v0.6.4/go.mod h1:kWv97z0sMAbnVNT/oe+JFZJVKn4xkas7ZdFf6UifWis=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF25UfsK2X7Ph/zCAk=
 github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
+github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
@@ -38,6 +101,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -62,5 +126,23 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y=
+k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU=
+k8s.io/apimachinery v0.20.6 h1:R5p3SlhaABYShQSO6LpPsYHjV05Q+79eBUR0Ut/f4tk=
+k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
+k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
+k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index cf3dc6b8e..ec1b5f7c1 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/dokku/dokku/plugins/common"
 	resty "github.com/go-resty/resty/v2"
@@ -75,6 +74,20 @@ func CommandInitialize() error {
 		return fmt.Errorf("Error installing k3s: %w", installerCmd.ExitError)
 	}
 
+	if err := common.TouchFile("/etc/rancher/k3s/registries.yaml"); err != nil {
+		return fmt.Errorf("Error creating initial registries.yaml file")
+	}
+
+	registryAclCmd := common.NewShellCmd(strings.Join([]string{
+		"setfacl",
+		"-m",
+		"user:dokku:rwx",
+		"/etc/rancher/k3s/registries.yaml",
+	}, " "))
+	if !registryAclCmd.Execute() {
+		return fmt.Errorf("Error updating acls on k3s registries.yaml file: %w", registryAclCmd.ExitError)
+	}
+
 	return nil
 }
 
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 95ba710dc..92252bbac 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -2,8 +2,11 @@ package scheduler_k3s
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/rancher/wharfie/pkg/registries"
+	"gopkg.in/yaml.v3"
 )
 
 // TriggerInstall runs the install step for the scheduler-k3s plugin
@@ -51,9 +54,47 @@ func TriggerPostDelete(appName string) error {
 }
 
 // TriggerPostRegistryLogin updates the `/etc/rancher/k3s/registries.yaml` to include
-// auth information for the registry
+// auth information for the registry. Note that if the file does not exist, it won't be updated.
 func TriggerPostRegistryLogin(server string, username string, password string) error {
-	// todo: setup auth
+	if !common.FileExists("/usr/local/bin/k3s") {
+		return nil
+	}
+
+	registry := registries.Registry{}
+	registryFile := "/etc/rancher/k3s/registries.yaml"
+	yamlFile, err := os.ReadFile(registryFile)
+	if err != nil {
+		return fmt.Errorf("Unable to read existing registries.yaml: %w", err)
+	}
+
+	err = yaml.Unmarshal(yamlFile, registry)
+	if err != nil {
+		return fmt.Errorf("Unable to unmarshal registry configuration from yaml: %w", err)
+	}
+
+	common.LogInfo1("Updating k3s configuration")
+	if registry.Auths == nil {
+		registry.Auths = map[string]registries.AuthConfig{}
+	}
+
+	if server == "docker.io" {
+		server = "registry-1.docker.io"
+	}
+
+	registry.Auths[server] = registries.AuthConfig{
+		Username: username,
+		Password: password,
+	}
+
+	data, err := yaml.Marshal(&registry)
+	if err != nil {
+		return fmt.Errorf("Unable to marshal registry configuration to yaml: %w", err)
+	}
+
+	if err := os.WriteFile(registryFile, data, os.FileMode(644)); err != nil {
+		return fmt.Errorf("Unable to write registry configuration to file: %w", err)
+	}
+
 	// todo: auth against all nodes in cluster
 	return nil
 }

From 7101d1bf7b5abc72d308ed01225c3b913a855298 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 12 Jan 2024 23:57:30 -0500
Subject: [PATCH 008/113] fix: close the file before attempting to write it

Also remove the unnecessary sleep call
---
 plugins/scheduler-k3s/subcommands.go | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index ec1b5f7c1..5baa7c511 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -31,6 +31,10 @@ func CommandInitialize() error {
 	}
 	defer os.Remove(f.Name())
 
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("Unable to close k3s installer file: %w", err)
+	}
+
 	err = common.WriteSliceToFile(common.WriteSliceToFileInput{
 		Filename: f.Name(),
 		Lines:    strings.Split(resp.String(), "\n"),
@@ -40,10 +44,6 @@ func CommandInitialize() error {
 		return fmt.Errorf("Unable to write k3s installer to file: %w", err)
 	}
 
-	if err := f.Close(); err != nil {
-		return fmt.Errorf("Unable to close k3s installer file: %w", err)
-	}
-
 	fi, err := os.Stat(f.Name())
 	if err != nil {
 		return fmt.Errorf("Unable to get k3s installer file size: %w", err)
@@ -53,8 +53,6 @@ func CommandInitialize() error {
 		return fmt.Errorf("Invalid k3s installer filesize")
 	}
 
-	time.Sleep(1 * time.Second)
-
 	// todo: allow this to be passed as an option or environment variable
 	token := "password"
 	if err := CommandSet("--global", "token", token); err != nil {

From 6c234c66a3a9e1e29705ba7a91dc51bf259d5021 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 12 Jan 2024 23:57:43 -0500
Subject: [PATCH 009/113] fix: guard the initialize call if k3s is already
 installed

---
 plugins/scheduler-k3s/subcommands.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 5baa7c511..8ea510773 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -11,6 +11,10 @@ import (
 
 // CommandInitialize initializes a k3s cluster on the local server
 func CommandInitialize() error {
+	if common.FileExists("/usr/local/bin/k3s") {
+		return fmt.Errorf("k3s already installed, cannot re-initialize k3s")
+	}
+
 	client := resty.New()
 	resp, err := client.R().
 		Get("https://get.k3s.io")

From 6fb03eda4e4569b97c4be54e65d45d6e8fb1e620 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 18:06:18 -0500
Subject: [PATCH 010/113] feat: implement most of deployments

The big feature missing from deployments today is healthchecks, which require extra parsing of the app.json format since the healthchecks there aren't immediately serializable to kubernetes deployment healthchecks.

Other smaller missing functionality from the existing OSS kubernetes scheduler are:

- The ability to set custom pod annotations
- The ability to set custom deployment annotations
- The ability to mount volumes

Exposing these pieces of functionality will come at a later date, if at all.
---
 go.work                                       |   1 +
 plugins/scheduler-k3s/Makefile                |   2 +-
 plugins/scheduler-k3s/functions.go            | 164 +++++
 plugins/scheduler-k3s/go.mod                  | 150 ++++-
 plugins/scheduler-k3s/go.sum                  | 563 ++++++++++++++++--
 plugins/scheduler-k3s/helm.go                 | 230 +++++++
 plugins/scheduler-k3s/scheduler_k3s.go        |  49 +-
 .../scheduler-k3s/src/triggers/triggers.go    |   5 +
 plugins/scheduler-k3s/subcommands.go          |   2 +-
 plugins/scheduler-k3s/template.go             | 443 ++++++++++++++
 plugins/scheduler-k3s/triggers.go             | 258 +++++++-
 11 files changed, 1810 insertions(+), 57 deletions(-)
 create mode 100644 plugins/scheduler-k3s/functions.go
 create mode 100644 plugins/scheduler-k3s/helm.go
 create mode 100644 plugins/scheduler-k3s/template.go

diff --git a/go.work b/go.work
index b0eb4a0e6..229ef2363 100644
--- a/go.work
+++ b/go.work
@@ -20,4 +20,5 @@ use (
 	./plugins/resource
 	./plugins/scheduler
 	./plugins/scheduler-docker-local
+	./plugins/scheduler-k3s
 )
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 3f64fb883..5b65cfbfa 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
new file mode 100644
index 000000000..dad531e83
--- /dev/null
+++ b/plugins/scheduler-k3s/functions.go
@@ -0,0 +1,164 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/dokku/dokku/plugins/common"
+	"github.com/dokku/dokku/plugins/config"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"mvdan.cc/sh/v3/shell"
+)
+
+// StartCommandInput contains all the information needed to get the start command
+type StartCommandInput struct {
+	// AppName is the name of the app
+	AppName string
+	// ProcessType is the process type
+	ProcessType string
+	// ImageSourceType is the image source type
+	ImageSourceType string
+	// Port is the port
+	Port int32
+	// Env is the environment variables
+	Env map[string]string
+}
+
+// StartCommandOutput contains the start command
+type StartCommandOutput struct {
+	// Command is the start command
+	Command []string
+}
+
+func NewClient() (*kubernetes.Clientset, error) {
+	config, err := clientcmd.BuildConfigFromFlags("", KubeConfigPath)
+	if err != nil {
+		return &kubernetes.Clientset{}, err
+	}
+
+	return kubernetes.NewForConfig(config)
+}
+
+func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
+	client, err := NewClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, namespace := range namespaces.Items {
+		if namespace.Name == namespaceName {
+			return nil
+		}
+	}
+
+	namespace := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: namespaceName,
+		},
+	}
+	_, err = client.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func extractStartCommand(input StartCommandInput) string {
+	command := ""
+	if input.ImageSourceType == "herokuish" {
+		command = "/start " + input.ProcessType
+	}
+
+	startCommandOverride, ok := config.Get(input.AppName, "DOKKU_START_CMD")
+	if ok {
+		command = startCommandOverride
+	}
+
+	if input.ImageSourceType == "herokuish" {
+		return command
+	}
+
+	if input.ImageSourceType == "dockerfile" {
+		startCommandOverride, ok := config.Get(input.AppName, "DOKKU_DOCKERFILE_START_CMD")
+		if ok {
+			command = startCommandOverride
+		}
+	}
+
+	if command == "" {
+		procfileStartCommand, _ := common.PlugnTriggerOutputAsString("procfile-get-command", []string{input.AppName, input.ProcessType, fmt.Sprint(input.Port)}...)
+		if procfileStartCommand != "" {
+			command = procfileStartCommand
+		}
+	}
+
+	return command
+}
+
+type PortMap struct {
+	ContainerPort int32  `json:"container_port"`
+	HostPort      int32  `json:"host_port"`
+	Scheme        string `json:"scheme"`
+}
+
+func (p PortMap) IsAllowedHttp() bool {
+	return p.Scheme == "http" || p.ContainerPort == 80
+}
+
+func (p PortMap) IsAllowedHttps() bool {
+	return p.Scheme == "https" || p.ContainerPort == 443
+}
+
+func getPortMaps(appName string) ([]PortMap, error) {
+	portMaps := []PortMap{}
+
+	output, err := common.PlugnTriggerOutputAsString("ports-get", []string{appName, "json"}...)
+	if err != nil {
+		return portMaps, err
+	}
+
+	err = json.Unmarshal([]byte(output), &portMaps)
+	if err != nil {
+		return portMaps, err
+	}
+
+	allowedMappings := []PortMap{}
+	for _, portMap := range portMaps {
+		if !portMap.IsAllowedHttp() && !portMap.IsAllowedHttps() {
+			// todo: log warning
+			continue
+		}
+
+		allowedMappings = append(allowedMappings, portMap)
+	}
+
+	return allowedMappings, nil
+}
+
+func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
+	command := extractStartCommand(input)
+	fields, err := shell.Fields(command, func(name string) string {
+		if name == "PORT" {
+			return fmt.Sprint(input.Port)
+		}
+
+		return input.Env[name]
+	})
+	if err != nil {
+		return StartCommandOutput{}, err
+	}
+
+	return StartCommandOutput{
+		Command: fields,
+	}, nil
+}
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 6b9d42898..4894316db 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -4,35 +4,165 @@ go 1.21
 
 require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
+	github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa
 	github.com/go-resty/resty/v2 v2.11.0
-	github.com/rancher/wharfie v0.6.4
+	github.com/rancher/wharfie v0.6.5
 	github.com/spf13/pflag v1.0.5
+	github.com/traefik/traefik/v2 v2.10.7
+	github.com/wk8/go-ordered-map/v2 v2.1.8
 	gopkg.in/yaml.v3 v3.0.1
+	helm.sh/helm/v3 v3.13.3
+	k8s.io/api v0.29.0
+	k8s.io/apimachinery v0.29.0
+	k8s.io/cli-runtime v0.29.0
+	k8s.io/client-go v0.29.0
+	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
+	mvdan.cc/sh/v3 v3.7.0
 )
 
 require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/Masterminds/squirrel v1.5.4 // indirect
+	github.com/Microsoft/hcsshim v0.11.0 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
 	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
-	github.com/docker/cli v20.10.20+incompatible // indirect
+	github.com/containerd/containerd v1.7.6 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/docker/cli v24.0.6+incompatible // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/docker/docker v20.10.26+incompatible // indirect
+	github.com/docker/docker v24.0.7+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
-	github.com/google/go-containerregistry v0.12.2-0.20230106184643-b063f6aeac72 // indirect
-	github.com/klauspost/compress v1.15.11 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/go-acme/lego/v4 v4.14.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-containerregistry v0.14.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.3.1 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gosuri/uitable v0.0.4 // indirect
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmoiron/sqlx v1.3.5 // indirect
+	github.com/joho/godotenv v1.2.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.1 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/miekg/dns v1.1.55 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/otiai10/copy v1.14.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rubenv/sql-migrate v1.5.2 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryanuber/columnize v2.1.2+incompatible // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/cobra v1.7.0 // indirect
+	github.com/traefik/paerser v0.2.0 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
+	go.opentelemetry.io/otel v1.14.0 // indirect
+	go.opentelemetry.io/otel/trace v1.14.0 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/grpc v1.58.3 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/apiextensions-apiserver v0.28.4 // indirect
+	k8s.io/apiserver v0.28.4 // indirect
+	k8s.io/component-base v0.28.4 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/kubectl v0.28.4 // indirect
+	oras.land/oras-go v1.2.4 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace github.com/dokku/dokku/plugins/common => ../common
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index 4fa3487e9..67ba25a3a 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -1,120 +1,532 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
+github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
+github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
+github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
+github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
+github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM=
+github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM=
+github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
+github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
+github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
+github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
+github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
+github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
+github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
+github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
+github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
+github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
-github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0=
-github.com/containerd/stargz-snapshotter/estargz v0.12.1/go.mod h1:12VUuCq3qPq4y8yUW+l5w3+oXV3cx2Po3KSe/SmPGqw=
+github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
+github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
+github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8=
+github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4=
+github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM=
+github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
+github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
+github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docker/cli v20.10.20+incompatible h1:lWQbHSHUFs7KraSN2jOJK7zbMS2jNCHI4mt4xUFUVQ4=
-github.com/docker/cli v20.10.20+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc=
+github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI=
+github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
+github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.26+incompatible h1:yhVdEnGdYdwRo2RAGciZU3P55Y7XH/ybb5A3L1d1rDE=
-github.com/docker/docker v20.10.26+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
 github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
+github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
+github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
+github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
+github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa h1:lIscxUTyCvJSHV9oY3JBQazshpvSc7H2kEBwIZddMfc=
+github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa/go.mod h1:7lDnO8LtGfKX1eFKAXunqGGTqwzJNaf6V7IkKhxlN2g=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
+github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
+github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
+github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
+github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
+github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
+github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/go-acme/lego/v4 v4.14.0 h1:/skZoRHgVh0d2RK7l1g3Ch8HqeqP9LB8ZEjLdGEpcDE=
+github.com/go-acme/lego/v4 v4.14.0/go.mod h1:zjmvNCDLGz7GrC1OqdVpVmZFKSRabEDtWbdzmcpBsGo=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs=
+github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw=
+github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
+github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
 github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU=
+github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs=
+github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0=
+github.com/gobuffalo/packd v1.0.1/go.mod h1:PP2POP3p3RXGz7Jh6eYEf93S7vA2za6xM7QT85L4+VY=
+github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY=
+github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
+github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
+github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.12.2-0.20230106184643-b063f6aeac72 h1:mjNVVKGHs3+xtaGPnTHz4ozBphZNsxsgqFR4+TNwRVM=
-github.com/google/go-containerregistry v0.12.2-0.20230106184643-b063f6aeac72/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo=
+github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
+github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo=
+github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
+github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
+github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
+github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
+github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
+github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/joho/godotenv v1.2.0 h1:vGTvz69FzUFp+X4/bAkb0j5BoLC+9bpqTWY8mjhA9pc=
+github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
-github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
+github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI=
+github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc=
+github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY=
+github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
+github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
+github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo=
+github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
+github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
+github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
+github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
+github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
+github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
-github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
+github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
+github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
 github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
 github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
+github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
+github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S+WV0JA6A=
 github.com/rancher/dynamiclistener v0.3.5/go.mod h1:dW/YF6/m2+uEyJ5VtEcd9THxda599HP6N9dSXk81+k0=
-github.com/rancher/wharfie v0.6.4 h1:JwYB+q661n8ut/ysgsjKe0P0z6bHCCFoC+29995ME90=
-github.com/rancher/wharfie v0.6.4/go.mod h1:kWv97z0sMAbnVNT/oe+JFZJVKn4xkas7ZdFf6UifWis=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rancher/wharfie v0.6.5 h1:Xxj/ki8d4BFeQBvyViW9jX8BqxaPlZwXvRD8BG2Mm/8=
+github.com/rancher/wharfie v0.6.5/go.mod h1:iGXn5Y9GEhsx1h/lPUXAklK4DoY4+6NOJF+Li1j248Y=
+github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97 h1:3RPlVWzZ/PDqmVuf/FKHARG5EMid/tl7cv54Sw/QRVY=
+github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=
+github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF25UfsK2X7Ph/zCAk=
 github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
+github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0=
+github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/traefik/paerser v0.2.0 h1:zqCLGSXoNlcBd+mzqSCLjon/I6phqIjeJL2xFB2ysgQ=
+github.com/traefik/paerser v0.2.0/go.mod h1:afzaVcgF8A+MpTnPG4wBr4whjanCSYA6vK5RwaYVtRc=
+github.com/traefik/traefik/v2 v2.10.7 h1:vs91abMRVFTxBjn9hsLkqPbQ+D0oM1+bf4K8nD5PKg4=
+github.com/traefik/traefik/v2 v2.10.7/go.mod h1:TD3saYNYbcBa7qDm29fIeyaiCMVX3ltv2YuaX3ANcnc=
 github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
+github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
+github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
+github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
+github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
+github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE=
+github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
+github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY=
+github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
+go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
+go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
+go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
+go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
+go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
+golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
@@ -122,27 +534,102 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E=
+golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
+google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y=
-k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU=
-k8s.io/apimachinery v0.20.6 h1:R5p3SlhaABYShQSO6LpPsYHjV05Q+79eBUR0Ut/f4tk=
-k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
-k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
-k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
+gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
+helm.sh/helm/v3 v3.13.3 h1:0zPEdGqHcubehJHP9emCtzRmu8oYsJFRrlVF3TFj8xY=
+helm.sh/helm/v3 v3.13.3/go.mod h1:3OKO33yI3p4YEXtTITN2+4oScsHeQe71KuzhlZ+aPfg=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
+k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
+k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU=
+k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM=
+k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
+k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
+k8s.io/apiserver v0.28.4 h1:BJXlaQbAU/RXYX2lRz+E1oPe3G3TKlozMMCZWu5GMgg=
+k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w=
+k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4=
+k8s.io/cli-runtime v0.29.0/go.mod h1:VKudXp3X7wR45L+nER85YUzOQIru28HQpXr0mTdeCrk=
+k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
+k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
+k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo=
+k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU=
+k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
+k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/kubectl v0.28.4 h1:gWpUXW/T7aFne+rchYeHkyB8eVDl5UZce8G4X//kjUQ=
+k8s.io/kubectl v0.28.4/go.mod h1:CKOccVx3l+3MmDbkXtIUtibq93nN2hkDR99XDCn7c/c=
+k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
+k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+mvdan.cc/sh/v3 v3.7.0 h1:lSTjdP/1xsddtaKfGg7Myu7DnlHItd3/M2tomOcNNBg=
+mvdan.cc/sh/v3 v3.7.0/go.mod h1:K2gwkaesF/D7av7Kxl0HbF5kGOd2ArupNTX3X44+8l8=
+oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=
+oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
+sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY=
+sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U=
+sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go
new file mode 100644
index 000000000..e5f462d6c
--- /dev/null
+++ b/plugins/scheduler-k3s/helm.go
@@ -0,0 +1,230 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+	"unicode"
+
+	"github.com/dokku/dokku/plugins/common"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/kube"
+	"helm.sh/helm/v3/pkg/storage/driver"
+)
+
+var DevNullPrinter = func(format string, v ...interface{}) {}
+
+var DeployLogPrinter = func(format string, v ...interface{}) {
+	message := strings.TrimSpace(fmt.Sprintf(format, v...))
+	if message == "" {
+		return
+	}
+	r := []rune(message)
+	r[0] = unicode.ToUpper(r[0])
+	s := string(r)
+
+	if strings.HasPrefix(s, "Beginning wait") {
+		common.LogExclaim(s)
+	} else if strings.HasPrefix(s, "Warning:") {
+		common.LogExclaim(s)
+	} else {
+		common.LogVerboseQuiet(s)
+	}
+}
+
+type HelmAgent struct {
+	Configuration *action.Configuration
+	Namespace     string
+	Logger        action.DebugLog
+}
+
+func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error) {
+	actionConfig, err := getActionConfig(namespace, logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return &HelmAgent{
+		Configuration: actionConfig,
+		Namespace:     namespace,
+	}, nil
+}
+
+func getActionConfig(namespace string, logger action.DebugLog) (*action.Configuration, error) {
+	actionConfig := new(action.Configuration)
+
+	helmDriver := os.Getenv("HELM_DRIVER")
+	if helmDriver == "" {
+		helmDriver = "secrets"
+	}
+
+	kubeConfig := kube.GetConfig(KubeConfigPath, "", namespace)
+	if err := actionConfig.Init(kubeConfig, namespace, helmDriver, logger); err != nil {
+		return nil, err
+	}
+	return actionConfig, nil
+}
+
+func (h *HelmAgent) ChartExists(releaseName string) (bool, error) {
+	client := action.NewHistory(h.Configuration)
+	client.Max = 1
+	releases, err := client.Run(releaseName)
+	if err != nil {
+		if errors.Is(err, driver.ErrReleaseNotFound) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	if len(releases) > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+type ChartInput struct {
+	ChartPath         string
+	Namespace         string
+	ReleaseName       string
+	RollbackOnFailure bool
+	Timeout           time.Duration
+	Values            map[string]interface{}
+}
+
+func (h *HelmAgent) InstallOrUpgradeChart(input ChartInput) error {
+	chartExists, err := h.ChartExists(input.ReleaseName)
+	if err != nil {
+		return err
+	}
+
+	if chartExists {
+		common.LogExclaim(fmt.Sprintf("Upgrading %s", input.ReleaseName))
+		return h.UpgradeChart(input)
+	}
+
+	common.LogExclaim(fmt.Sprintf("Installing %s", input.ReleaseName))
+	return h.InstallChart(input)
+}
+
+func (h *HelmAgent) InstallChart(input ChartInput) error {
+	namespace := input.Namespace
+	if namespace == "" {
+		namespace = h.Namespace
+	}
+
+	if input.ChartPath == "" {
+		return fmt.Errorf("Chart path is required")
+	}
+	if input.ReleaseName == "" {
+		return fmt.Errorf("Release name is required")
+	}
+	if input.Values == nil {
+		input.Values = map[string]interface{}{}
+	}
+
+	client := action.NewInstall(h.Configuration)
+	client.Atomic = false
+	client.ChartPathOptions = action.ChartPathOptions{}
+	client.CreateNamespace = true
+	client.DryRun = false
+	client.Namespace = namespace
+	client.ReleaseName = input.ReleaseName
+	client.Timeout = input.Timeout
+	client.Wait = false
+
+	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil)
+	if err != nil {
+		return fmt.Errorf("Error locating chart: %w", err)
+	}
+
+	chartRequested, err := loader.Load(chart)
+	if err != nil {
+		return fmt.Errorf("Error loading chart: %w", err)
+	}
+
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	cSignal := make(chan os.Signal, 2)
+	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-cSignal
+		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", input.ReleaseName))
+		cancel()
+	}()
+
+	_, err = client.RunWithContext(ctx, chartRequested, input.Values)
+	if err != nil {
+		return fmt.Errorf("Error deploying: %w", err)
+	}
+
+	return nil
+}
+
+func (h *HelmAgent) UpgradeChart(input ChartInput) error {
+	namespace := input.Namespace
+	if namespace == "" {
+		namespace = h.Namespace
+	}
+
+	if input.ChartPath == "" {
+		return fmt.Errorf("Chart path is required")
+	}
+	if input.ReleaseName == "" {
+		return fmt.Errorf("Release name is required")
+	}
+	if input.Values == nil {
+		input.Values = map[string]interface{}{}
+	}
+
+	client := action.NewUpgrade(h.Configuration)
+	client.Atomic = input.RollbackOnFailure
+	client.ChartPathOptions = action.ChartPathOptions{}
+	client.DryRun = false
+	client.Namespace = namespace
+	client.Timeout = input.Timeout
+	client.Wait = false
+
+	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil)
+	if err != nil {
+		return fmt.Errorf("Error locating chart: %w", err)
+	}
+
+	chartRequested, err := loader.Load(chart)
+	if err != nil {
+		return fmt.Errorf("Error loading chart: %w", err)
+	}
+
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	cSignal := make(chan os.Signal, 2)
+	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-cSignal
+		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", input.ReleaseName))
+		cancel()
+	}()
+
+	_, err = client.RunWithContext(ctx, input.ReleaseName, chartRequested, input.Values)
+	if err != nil {
+		return fmt.Errorf("Error deploying: %w", err)
+	}
+
+	return nil
+}
+
+func (h *HelmAgent) UninstallChart(releaseName string) error {
+	uninstall := action.NewUninstall(h.Configuration)
+	_, err := uninstall.Run(releaseName)
+	if err != nil {
+		return fmt.Errorf("Error uninstalling chart: %w", err)
+	}
+
+	return nil
+}
diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 88b5df958..6837b91fe 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -1,12 +1,49 @@
 package scheduler_k3s
 
-var (
-	// DefaultProperties is a map of all valid network properties with corresponding default property values
-	DefaultProperties = map[string]string{}
+import (
+	"sync"
 
-	// GlobalProperties is a map of all valid global network properties
+	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	kjson "k8s.io/apimachinery/pkg/runtime/serializer/json"
+)
+
+var (
+	// DefaultProperties is a map of all valid k3s properties with corresponding default property values
+	DefaultProperties = map[string]string{
+		"image-pull-secrets":  "",
+		"rollback-on-failure": "",
+		"namespace":           "default",
+		"timeout":             "300",
+	}
+
+	// GlobalProperties is a map of all valid global k3s properties
 	GlobalProperties = map[string]bool{
-		"network-interface": true,
-		"token":             true,
+		"network-interface":   true,
+		"rollback-on-failure": true,
+		"token":               true,
+		"timeout":             true,
 	}
 )
+
+const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
+
+var (
+	runtimeScheme  = runtime.NewScheme()
+	codecs         = serializer.NewCodecFactory(runtimeScheme)
+	deserializer   = codecs.UniversalDeserializer()
+	jsonSerializer = kjson.NewSerializerWithOptions(kjson.DefaultMetaFactory, runtimeScheme, runtimeScheme, kjson.SerializerOptions{})
+)
+
+var k8sNativeSchemeOnce sync.Once
+
+func init() {
+	k8sNativeSchemeOnce.Do(func() {
+		_ = appsv1.AddToScheme(runtimeScheme)
+		_ = corev1.AddToScheme(runtimeScheme)
+		_ = traefikv1alpha1.AddToScheme(runtimeScheme)
+	})
+}
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index 1d9ab3b90..4f070ba51 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -40,6 +40,11 @@ func main() {
 	case "report":
 		appName := flag.Arg(0)
 		err = scheduler_k3s.ReportSingleApp(appName, "", "")
+	case "scheduler-deploy":
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		imageTag := flag.Arg(2)
+		err = scheduler_k3s.TriggerSchedulerDeploy(scheduler, appName, imageTag)
 	default:
 		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
 	}
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 8ea510773..ed28374a3 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -42,7 +42,7 @@ func CommandInitialize() error {
 	err = common.WriteSliceToFile(common.WriteSliceToFileInput{
 		Filename: f.Name(),
 		Lines:    strings.Split(resp.String(), "\n"),
-		Mode:     os.FileMode(755),
+		Mode:     os.FileMode(0755),
 	})
 	if err != nil {
 		return fmt.Errorf("Unable to write k3s installer to file: %w", err)
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
new file mode 100644
index 000000000..adb40491f
--- /dev/null
+++ b/plugins/scheduler-k3s/template.go
@@ -0,0 +1,443 @@
+package scheduler_k3s
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
+	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
+	orderedmap "github.com/wk8/go-ordered-map/v2"
+	"gopkg.in/yaml.v3"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/cli-runtime/pkg/printers"
+	"k8s.io/utils/ptr"
+)
+
+type Chart struct {
+	ApiVersion string `yaml:"apiVersion"`
+	AppVersion string `yaml:"appVersion"`
+	Name       string `yaml:"name"`
+	Version    string `yaml:"version"`
+}
+
+type CreateIngressRoutesInput struct {
+	AppName    string
+	ChartDir   string
+	Deployment appsv1.Deployment
+	Namespace  string
+	PortMaps   []PortMap
+	Service    v1.Service
+}
+
+type Deployment struct {
+	AppName          string
+	Command          []string
+	Image            string
+	ImagePullSecrets string
+	ImageSourceType  string
+	Namespace        string
+	PrimaryPort      int32
+	PortMaps         []PortMap
+	ProcessType      string
+	Replicas         int32
+}
+
+type IngressRouteEntrypoint string
+
+const (
+	IngressRouteEntrypoint_HTTP  IngressRouteEntrypoint = "web"
+	IngressRouteEntrypoint_HTTPS IngressRouteEntrypoint = "websecure"
+)
+
+type IngressRoute struct {
+	AppName     string
+	Entrypoints []IngressRouteEntrypoint
+	Hostnames   []string
+	Namespace   string
+	Port        int32
+	PortMap     PortMap
+	ProcessType string
+	ServiceName string
+}
+
+type Secret struct {
+	AppName   string
+	Env       map[string]string
+	Namespace string
+}
+
+type Service struct {
+	AppName   string
+	Namespace string
+	Port      int32
+}
+
+type PrintInput struct {
+	AppendContents string
+	Object         runtime.Object
+	Path           string
+	Name           string
+	Replacements   *orderedmap.OrderedMap[string, string]
+}
+
+type Values struct {
+	DeploymentID string                   `yaml:"deploment_id"`
+	Secrets      map[string]string        `yaml:"secrets"`
+	Processes    map[string]ValuesProcess `yaml:"processes"`
+}
+
+type ValuesProcess struct {
+	Replicas int32 `yaml:"replicas"`
+}
+
+type WriteYamlInput struct {
+	Object interface{}
+	Path   string
+}
+
+func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
+	err := common.PlugnTrigger("domains-vhost-enabled", []string{input.AppName}...)
+	isAppVhostEnabled := err == nil
+
+	if isAppVhostEnabled {
+		b, err := common.PlugnTriggerOutput("domains-list", []string{input.AppName}...)
+		if err != nil {
+			return fmt.Errorf("Error getting domains for deployment: %w", err)
+		}
+		domains := []string{}
+		for _, domain := range strings.Split(string(b), "\n") {
+			domain = strings.TrimSpace(domain)
+			if domain != "" {
+				domains = append(domains, domain)
+			}
+		}
+
+		if len(domains) == 0 {
+			return nil
+		}
+
+		for _, portMap := range input.PortMaps {
+			ingressRoute := templateKubernetesIngressRoute(IngressRoute{
+				AppName:     input.AppName,
+				Hostnames:   domains,
+				Namespace:   input.Namespace,
+				Port:        input.Deployment.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort,
+				PortMap:     portMap,
+				ProcessType: "web",
+				ServiceName: input.Service.Name,
+			})
+
+			err = printResource(PrintInput{
+				Object: &ingressRoute,
+				Path:   filepath.Join(input.ChartDir, fmt.Sprintf("templates/ingress-route-%s-%d-%d.yaml", portMap.Scheme, portMap.HostPort, portMap.ContainerPort)),
+				Name:   ingressRoute.Name,
+			})
+			if err != nil {
+				return fmt.Errorf("Error printing ingress route: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func printResource(input PrintInput) error {
+	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
+	printr := printers.NewTypeSetter(runtimeScheme).ToPrinter(&printers.YAMLPrinter{})
+	handle, err := os.Create(input.Path)
+	if err != nil {
+		return fmt.Errorf("Error creating template file: %w", err)
+	}
+	defer handle.Close()
+
+	if err := printr.PrintObj(input.Object, handle); err != nil {
+		return fmt.Errorf("Error writing template file: %w", err)
+	}
+
+	if input.Replacements != nil {
+		b, err := os.ReadFile(input.Path)
+		if err != nil {
+			return fmt.Errorf("Error reading template file: %w", err)
+		}
+
+		contents := string(b)
+		for pair := input.Replacements.Oldest(); pair != nil; pair = pair.Next() {
+			contents = strings.ReplaceAll(string(contents), pair.Key, pair.Value)
+		}
+
+		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
+		if err != nil {
+			return fmt.Errorf("Error updating template file with replacements: %w", err)
+		}
+	}
+
+	if input.AppendContents != "" {
+		b, err := os.ReadFile(input.Path)
+		if err != nil {
+			return fmt.Errorf("Error reading template file: %w", err)
+		}
+
+		contents := string(b) + "\n" + input.AppendContents
+
+		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
+		if err != nil {
+			return fmt.Errorf("Error updating template file with replacements: %w", err)
+		}
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(input.Path)
+	}
+	return nil
+}
+
+func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
+	labels := map[string]string{
+		"app":              input.AppName,
+		"process-type":     input.ProcessType,
+		"app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+	}
+	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
+
+	deployment := appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+			Namespace: input.Namespace,
+			Labels:    labels,
+			Annotations: map[string]string{
+				"dokku.com/app-name":      input.AppName,
+				"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
+				"dokku.com/process-type":  input.ProcessType,
+				"dokku.com/builder-type":  input.ImageSourceType,
+			},
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas:             ptr.To(input.Replicas),
+			RevisionHistoryLimit: ptr.To(int32(5)),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+							Env:  []corev1.EnvVar{},
+							EnvFrom: []corev1.EnvFromSource{
+								{
+									SecretRef: &corev1.SecretEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: secretName,
+										},
+										Optional: ptr.To(true),
+									},
+								},
+							},
+							Image:           input.Image,
+							ImagePullPolicy: corev1.PullPolicy("Always"),
+							Resources: corev1.ResourceRequirements{
+								Limits:   corev1.ResourceList{},
+								Requests: corev1.ResourceList{},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if len(input.Command) > 0 {
+		deployment.Spec.Template.Spec.Containers[0].Args = input.Command
+	}
+
+	if input.ProcessType == "web" {
+		for _, portMap := range input.PortMaps {
+			protocol := "TCP"
+			if portMap.Scheme == "udp" {
+				protocol = "UDP"
+			}
+			deployment.Spec.Template.Spec.Containers[0].Ports = append(deployment.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{
+				Name:          fmt.Sprintf("%s-%d-%d", portMap.Scheme, portMap.HostPort, portMap.ContainerPort),
+				ContainerPort: portMap.ContainerPort,
+				Protocol:      corev1.Protocol(protocol),
+			})
+		}
+
+		deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
+			Name:  "PORT",
+			Value: fmt.Sprint(input.PrimaryPort),
+		})
+	}
+
+	if input.ImagePullSecrets != "" {
+		deployment.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
+			{
+				Name: input.ImagePullSecrets,
+			},
+		}
+	}
+
+	cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
+	if err != nil && cpuLimit != "" && cpuLimit != "0" {
+		cpuQuantity, err := resource.ParseQuantity(cpuLimit)
+		if err != nil {
+			return deployment, fmt.Errorf("Error parsing cpu limit: %w", err)
+		}
+		deployment.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
+	}
+	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
+	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
+		nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
+		if err != nil {
+			return deployment, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
+		}
+		deployment.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
+	}
+	memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
+	if err != nil && memoryLimit != "" && memoryLimit != "0" {
+		memoryQuantity, err := resource.ParseQuantity(memoryLimit)
+		if err != nil {
+			return deployment, fmt.Errorf("Error parsing memory limit: %w", err)
+		}
+		deployment.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
+	}
+
+	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
+	if err != nil && cpuRequest != "" && cpuRequest != "0" {
+		cpuQuantity, err := resource.ParseQuantity(cpuRequest)
+		if err != nil {
+			return deployment, fmt.Errorf("Error parsing cpu request: %w", err)
+		}
+		deployment.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
+	}
+	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
+	if err != nil && memoryRequest != "" && memoryRequest != "0" {
+		memoryQuantity, err := resource.ParseQuantity(memoryRequest)
+		if err != nil {
+			return deployment, fmt.Errorf("Error parsing memory request: %w", err)
+		}
+		deployment.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
+	}
+
+	return deployment, nil
+}
+
+func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressRoute {
+	entryPoint := IngressRouteEntrypoint_HTTP
+	if input.PortMap.Scheme == "https" {
+		entryPoint = IngressRouteEntrypoint_HTTPS
+	}
+
+	port := fmt.Sprintf("%s-%d-%d", input.PortMap.Scheme, input.PortMap.HostPort, input.PortMap.ContainerPort)
+	ingressRoute := traefikv1alpha1.IngressRoute{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-%s", input.ServiceName, port),
+			Namespace: input.Namespace,
+			Labels: map[string]string{
+				"dokku.com/app":          input.AppName,
+				"dokku.com/process-type": input.ProcessType,
+			},
+		},
+		Spec: traefikv1alpha1.IngressRouteSpec{
+			EntryPoints: []string{string(entryPoint)},
+			Routes:      []traefikv1alpha1.Route{},
+		},
+	}
+
+	for _, hostname := range input.Hostnames {
+		rule := traefikv1alpha1.Route{
+			Kind:  "Rule",
+			Match: "Host(`" + hostname + "`)",
+			Services: []traefikv1alpha1.Service{
+				{
+					LoadBalancerSpec: traefikv1alpha1.LoadBalancerSpec{
+						Name:           input.ServiceName,
+						Namespace:      input.Namespace,
+						PassHostHeader: ptr.To(true),
+						Port:           intstr.FromString(port),
+						Scheme:         input.PortMap.Scheme,
+					},
+				},
+			},
+		}
+
+		ingressRoute.Spec.Routes = append(ingressRoute.Spec.Routes, rule)
+	}
+
+	return ingressRoute
+}
+
+func templateKubernetesSecret(input Secret) corev1.Secret {
+	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: input.Namespace,
+			Labels: map[string]string{
+				"dokku.com/app-name":      input.AppName,
+				"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
+			},
+		},
+		Data: map[string][]byte{},
+	}
+
+	return secret
+}
+
+func templateKubernetesService(input Service) corev1.Service {
+	service := corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-%s", input.AppName, "web"),
+			Namespace: input.Namespace,
+			Labels: map[string]string{
+				"dokku.com/app":              input.AppName,
+				"dokku.com/process-type":     "web",
+				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"dokku.com/app":              input.AppName,
+				"dokku.com/process-type":     "web",
+				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
+			},
+			Ports: []corev1.ServicePort{
+				{
+					Name:       "web",
+					Port:       input.Port,
+					TargetPort: intstr.FromInt32(input.Port),
+				},
+			},
+		},
+	}
+	return service
+}
+
+func writeYaml(input WriteYamlInput) error {
+	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
+	data, err := yaml.Marshal(input.Object)
+	if err != nil {
+		return fmt.Errorf("Error marshalling chart: %w", err)
+	}
+
+	err = os.WriteFile(input.Path, data, os.FileMode(0644))
+	if err != nil {
+		return fmt.Errorf("Error writing chart: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(input.Path)
+	}
+
+	return nil
+}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 92252bbac..a581d63ef 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -1,12 +1,20 @@
 package scheduler_k3s
 
 import (
+	"context"
+	"encoding/base64"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strconv"
+	"time"
 
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/dokku/dokku/plugins/config"
 	"github.com/rancher/wharfie/pkg/registries"
+	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
+	appsv1 "k8s.io/api/apps/v1"
 )
 
 // TriggerInstall runs the install step for the scheduler-k3s plugin
@@ -91,10 +99,258 @@ func TriggerPostRegistryLogin(server string, username string, password string) e
 		return fmt.Errorf("Unable to marshal registry configuration to yaml: %w", err)
 	}
 
-	if err := os.WriteFile(registryFile, data, os.FileMode(644)); err != nil {
+	if err := os.WriteFile(registryFile, data, os.FileMode(0644)); err != nil {
 		return fmt.Errorf("Unable to write registry configuration to file: %w", err)
 	}
 
 	// todo: auth against all nodes in cluster
 	return nil
 }
+
+// TriggerSchedulerDeploy deploys an image tag for a given application
+func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+	s, err := common.PlugnTriggerOutput("ps-current-scale", []string{appName}...)
+	if err != nil {
+		return err
+	}
+
+	processes, err := common.ParseScaleOutput(s)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	if err := createKubernetesNamespace(ctx, namespace); err != nil {
+		return fmt.Errorf("Error creating kubernetes namespace for deployment: %w", err)
+	}
+
+	image, err := common.GetDeployingAppImageName(appName, imageTag, "")
+	if err != nil {
+		return fmt.Errorf("Error getting deploying app image name: %w", err)
+	}
+
+	deployTimeout := common.PropertyGetDefault("scheduler-k3s", appName, "deploy-timeout", "300s")
+	if _, err := strconv.Atoi(deployTimeout); err == nil {
+		deployTimeout = fmt.Sprintf("%ss", deployTimeout)
+	}
+
+	deployRollback := common.PropertyGetDefault("scheduler-k3s", appName, "rollback-on-failure", "false")
+	allowRollbacks, err := strconv.ParseBool(deployRollback)
+	if err != nil {
+		return fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err)
+	}
+
+	imagePullSecrets := common.PropertyGetDefault("scheduler-k3s", appName, "image-pull-secrets", "")
+
+	imageSourceType := "dockerfile"
+	if common.IsImageCnbBased(image) {
+		imageSourceType = "pack"
+	} else if common.IsImageHerokuishBased(image, appName) {
+		imageSourceType = "herokuish"
+	}
+
+	env, err := config.LoadMergedAppEnv(appName)
+	if err != nil {
+		return fmt.Errorf("Error loading environment for deployment: %w", err)
+	}
+
+	chartDir, err := os.MkdirTemp("", "dokku-chart-")
+	if err != nil {
+		return fmt.Errorf("Error creating chart directory: %w", err)
+	}
+	defer os.RemoveAll(chartDir)
+
+	common.LogDebug(fmt.Sprintf("Using chart directory: %s", chartDir))
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
+		return fmt.Errorf("Error creating chart templates directory: %w", err)
+	}
+
+	deploymentId := time.Now().Unix()
+	replacements := orderedmap.New[string, string]()
+	replacements.Set("DEPLOYMENT_ID_QUOTED", "{{.Values.deploment_id | quote}}")
+	replacements.Set("DEPLOYMENT_ID", "{{.Values.deploment_id}}")
+
+	secret := templateKubernetesSecret(Secret{
+		AppName:   appName,
+		Env:       env.Map(),
+		Namespace: namespace,
+	})
+	err = printResource(PrintInput{
+		Object:       &secret,
+		Path:         filepath.Join(chartDir, "templates/secret.yaml"),
+		Name:         secret.Name,
+		Replacements: replacements,
+		AppendContents: `{{- with .Values.secrets }}
+data:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+`,
+	})
+	if err != nil {
+		return fmt.Errorf("Error printing deployment: %w", err)
+	}
+
+	portMaps, err := getPortMaps(appName)
+	if err != nil {
+		return fmt.Errorf("Error getting port mappings for deployment: %w", err)
+	}
+
+	primaryPort := int32(5000)
+	for _, portMap := range portMaps {
+		primaryPort = portMap.ContainerPort
+		if primaryPort != 0 {
+			break
+		}
+	}
+
+	deployments := map[string]appsv1.Deployment{}
+	i := 0
+	for processType := range processes {
+		startCommand, err := getStartCommand(StartCommandInput{
+			AppName:         appName,
+			ProcessType:     processType,
+			ImageSourceType: imageSourceType,
+			Port:            primaryPort,
+			Env:             env.Map(),
+		})
+		if err != nil {
+			return fmt.Errorf("Error getting start command for deployment: %w", err)
+		}
+
+		i++
+		replicaCountPlaceholder := int32(i * 1000)
+
+		// todo: implement healthchecks
+		// todo: implement deployment annotations
+		// todo: implement pod annotations
+		// todo: implement volumes
+		deployment, err := templateKubernetesDeployment(Deployment{
+			AppName:          appName,
+			Command:          startCommand.Command,
+			Image:            image,
+			ImagePullSecrets: imagePullSecrets,
+			ImageSourceType:  imageSourceType,
+			Namespace:        namespace,
+			PrimaryPort:      primaryPort,
+			PortMaps:         portMaps,
+			ProcessType:      processType,
+			Replicas:         replicaCountPlaceholder,
+		})
+		if err != nil {
+			return fmt.Errorf("Error templating deployment: %w", err)
+		}
+
+		replacements.Set(fmt.Sprintf("replicas: %d", replicaCountPlaceholder), fmt.Sprintf("replicas: {{.Values.processes.%s.replicas}}", processType))
+		deployments[processType] = deployment
+		err = printResource(PrintInput{
+			Object:       &deployment,
+			Path:         filepath.Join(chartDir, fmt.Sprintf("templates/deployment-%s.yaml", deployment.Name)),
+			Name:         deployment.Name,
+			Replacements: replacements,
+		})
+		replacements.Delete(fmt.Sprintf("replicas: %d", replicaCountPlaceholder))
+
+		if err != nil {
+			return fmt.Errorf("Error printing deployment: %w", err)
+		}
+	}
+
+	if deployment, ok := deployments["web"]; ok {
+		service := templateKubernetesService(Service{
+			AppName:   appName,
+			Namespace: namespace,
+			Port:      deployment.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort,
+		})
+
+		err := printResource(PrintInput{
+			Object:       &service,
+			Path:         filepath.Join(chartDir, "templates/service-web.yaml"),
+			Name:         service.Name,
+			Replacements: replacements,
+		})
+		if err != nil {
+			return fmt.Errorf("Error printing service: %w", err)
+		}
+
+		err = createIngressRoutesFiles(CreateIngressRoutesInput{
+			AppName:    appName,
+			ChartDir:   chartDir,
+			Deployment: deployment,
+			Namespace:  namespace,
+			PortMaps:   portMaps,
+			Service:    service,
+		})
+		if err != nil {
+			return fmt.Errorf("Error creating ingress routes: %w", err)
+		}
+	}
+
+	chart := &Chart{
+		ApiVersion: "v2",
+		AppVersion: "1.0.0",
+		Name:       appName,
+		Version:    fmt.Sprintf("0.0.%d", deploymentId),
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: chart,
+		Path:   filepath.Join(chartDir, "Chart.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("Error writing chart: %w", err)
+	}
+
+	values := &Values{
+		DeploymentID: fmt.Sprint(deploymentId),
+		Secrets:      map[string]string{},
+		Processes:    map[string]ValuesProcess{},
+	}
+	for processType, processCount := range processes {
+		values.Processes[processType] = ValuesProcess{
+			Replicas: int32(processCount),
+		}
+	}
+	for key, value := range env.Map() {
+		values.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value))
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: values,
+		Path:   filepath.Join(chartDir, "values.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("Error writing chart: %w", err)
+	}
+
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("Error creating helm agent: %w", err)
+	}
+
+	chartPath, err := filepath.Abs(chartDir)
+	if err != nil {
+		return fmt.Errorf("Error getting chart path: %w", err)
+	}
+
+	timeoutDuration, err := time.ParseDuration(deployTimeout)
+	if err != nil {
+		return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
+	}
+
+	err = helmAgent.InstallOrUpgradeChart(ChartInput{
+		ChartPath:         chartPath,
+		Namespace:         namespace,
+		ReleaseName:       fmt.Sprintf("dokku-%s", appName),
+		RollbackOnFailure: allowRollbacks,
+		Timeout:           timeoutDuration,
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

From 737f54a5c714968698a3447b39fbf3e4c02c9256 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 20:56:30 -0500
Subject: [PATCH 011/113] feat: implement dokku enter for k8s scheduler

---
 docs/processes/entering-containers.md         |   7 +
 plugins/scheduler-k3s/Makefile                |   2 +-
 plugins/scheduler-k3s/functions.go            |  52 +++++-
 plugins/scheduler-k3s/go.mod                  |   4 +-
 .../scheduler-k3s/src/triggers/triggers.go    |  14 ++
 plugins/scheduler-k3s/template.go             |  34 ++--
 plugins/scheduler-k3s/triggers.go             | 148 ++++++++++++++++++
 7 files changed, 235 insertions(+), 26 deletions(-)

diff --git a/docs/processes/entering-containers.md b/docs/processes/entering-containers.md
index 4d663155d..06a3804d2 100644
--- a/docs/processes/entering-containers.md
+++ b/docs/processes/entering-containers.md
@@ -40,3 +40,10 @@ dokku enter node-js-app web echo hi
 # run a long-running command, as one might for a cron task
 dokku enter node-js-app web python script/background-worker.py
 ```
+
+If the custom command uses flags, it must be separated from the `dokku enter` command via `--` (double dashes):
+
+```shell
+# list files in the app
+dokku enter node-js-app web -- ls -lah
+```
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 5b65cfbfa..b12cc544a 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index dad531e83..c8ec555e4 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -9,8 +9,12 @@ import (
 	"github.com/dokku/dokku/plugins/config"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"mvdan.cc/sh/v3/shell"
 )
 
@@ -34,22 +38,56 @@ type StartCommandOutput struct {
 	Command []string
 }
 
-func NewClient() (*kubernetes.Clientset, error) {
-	config, err := clientcmd.BuildConfigFromFlags("", KubeConfigPath)
+type KubernetesClient struct {
+	Client     kubernetes.Clientset
+	RestClient rest.Interface
+	RestConfig rest.Config
+}
+
+func NewKubernetesClient() (KubernetesClient, error) {
+	clientConfig := KubernetesClientConfig()
+	restConf, err := clientConfig.ClientConfig()
 	if err != nil {
-		return &kubernetes.Clientset{}, err
+		return KubernetesClient{}, err
 	}
 
-	return kubernetes.NewForConfig(config)
+	restConf.GroupVersion = &schema.GroupVersion{
+		Group:   "api",
+		Version: "v1",
+	}
+
+	client, err := kubernetes.NewForConfig(restConf)
+	if err != nil {
+		return KubernetesClient{}, err
+	}
+
+	restConf.NegotiatedSerializer = runtime.NewSimpleNegotiatedSerializer(runtime.SerializerInfo{})
+
+	restClient, err := rest.RESTClientFor(restConf)
+	if err != nil {
+		return KubernetesClient{}, err
+	}
+
+	return KubernetesClient{
+		Client:     *client,
+		RestConfig: *restConf,
+		RestClient: restClient,
+	}, nil
+}
+
+func KubernetesClientConfig() clientcmd.ClientConfig {
+	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: KubeConfigPath},
+		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
 }
 
 func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
-	client, err := NewClient()
+	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return err
 	}
 
-	namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	namespaces, err := clientset.Client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return err
 	}
@@ -65,7 +103,7 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 			Name: namespaceName,
 		},
 	}
-	_, err = client.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{})
+	_, err = clientset.Client.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{})
 	if err != nil {
 		return err
 	}
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 4894316db..0fb4be71f 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -16,6 +16,8 @@ require (
 	k8s.io/apimachinery v0.29.0
 	k8s.io/cli-runtime v0.29.0
 	k8s.io/client-go v0.29.0
+	k8s.io/klog/v2 v2.110.1
+	k8s.io/kubectl v0.28.4
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	mvdan.cc/sh/v3 v3.7.0
 )
@@ -154,9 +156,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	k8s.io/kubectl v0.28.4 // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index 4f070ba51..f5095470e 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	flag "github.com/spf13/pflag"
+	"k8s.io/utils/ptr"
 
 	"github.com/dokku/dokku/plugins/common"
 	scheduler_k3s "github.com/dokku/dokku/plugins/scheduler-k3s"
@@ -15,6 +16,7 @@ import (
 func main() {
 	parts := strings.Split(os.Args[0], "/")
 	trigger := parts[len(parts)-1]
+	podIdentifier := flag.String("container-id", "", "--container-id: A pod identifier")
 	flag.Parse()
 
 	var err error
@@ -45,6 +47,18 @@ func main() {
 		appName := flag.Arg(1)
 		imageTag := flag.Arg(2)
 		err = scheduler_k3s.TriggerSchedulerDeploy(scheduler, appName, imageTag)
+	case "scheduler-enter":
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		containerType := flag.Arg(2)
+		args := flag.Args()
+		if len(args) > 3 {
+			_, args = common.ShiftString(args)
+			_, args = common.ShiftString(args)
+			_, args = common.ShiftString(args)
+		}
+
+		err = scheduler_k3s.TriggerSchedulerEnter(scheduler, appName, containerType, ptr.Deref(podIdentifier, ""), args)
 	default:
 		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
 	}
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index adb40491f..154baafb1 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -200,23 +200,24 @@ func printResource(input PrintInput) error {
 
 func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	labels := map[string]string{
-		"app":              input.AppName,
-		"process-type":     input.ProcessType,
-		"app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"dokku.com/app-name":         input.AppName,
+		"dokku.com/process-type":     input.ProcessType,
+		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+	}
+	annotations := map[string]string{
+		"dokku.com/app-name":      input.AppName,
+		"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
+		"dokku.com/process-type":  input.ProcessType,
+		"dokku.com/builder-type":  input.ImageSourceType,
 	}
 	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
 
 	deployment := appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-			Namespace: input.Namespace,
-			Labels:    labels,
-			Annotations: map[string]string{
-				"dokku.com/app-name":      input.AppName,
-				"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
-				"dokku.com/process-type":  input.ProcessType,
-				"dokku.com/builder-type":  input.ImageSourceType,
-			},
+			Name:        fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+			Namespace:   input.Namespace,
+			Labels:      labels,
+			Annotations: annotations,
 		},
 		Spec: appsv1.DeploymentSpec{
 			Replicas:             ptr.To(input.Replicas),
@@ -226,7 +227,8 @@ func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: labels,
+					Labels:      labels,
+					Annotations: annotations,
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
@@ -344,7 +346,7 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 			Name:      fmt.Sprintf("%s-%s", input.ServiceName, port),
 			Namespace: input.Namespace,
 			Labels: map[string]string{
-				"dokku.com/app":          input.AppName,
+				"dokku.com/app-name":     input.AppName,
 				"dokku.com/process-type": input.ProcessType,
 			},
 		},
@@ -400,14 +402,14 @@ func templateKubernetesService(input Service) corev1.Service {
 			Name:      fmt.Sprintf("%s-%s", input.AppName, "web"),
 			Namespace: input.Namespace,
 			Labels: map[string]string{
-				"dokku.com/app":              input.AppName,
+				"dokku.com/app-name":         input.AppName,
 				"dokku.com/process-type":     "web",
 				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
 			},
 		},
 		Spec: corev1.ServiceSpec{
 			Selector: map[string]string{
-				"dokku.com/app":              input.AppName,
+				"dokku.com/app-name":         input.AppName,
 				"dokku.com/process-type":     "web",
 				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
 			},
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index a581d63ef..c753d6b61 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -5,8 +5,12 @@ import (
 	"encoding/base64"
 	"fmt"
 	"os"
+	"os/signal"
 	"path/filepath"
+	"slices"
 	"strconv"
+	"strings"
+	"syscall"
 	"time"
 
 	"github.com/dokku/dokku/plugins/common"
@@ -15,6 +19,11 @@ import (
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/kubectl/pkg/util/term"
 )
 
 // TriggerInstall runs the install step for the scheduler-k3s plugin
@@ -354,3 +363,142 @@ data:
 
 	return nil
 }
+
+// TriggerSchedulerEnter enters a container for a given application
+func TriggerSchedulerEnter(scheduler string, appName string, processType string, podName string, args []string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+
+	common.LogDebug(fmt.Sprintf("%s %s %s %s", scheduler, appName, processType, podName))
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	cSignal := make(chan os.Signal, 2)
+	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-cSignal
+		common.LogWarn("Exiting pod")
+		cancel()
+	}()
+
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+
+	labelSelector := []string{fmt.Sprintf("dokku.com/app-name=%s", appName)}
+	processIndex := 1
+	if processType != "" {
+		parts := strings.SplitN(processType, ".", 2)
+		if len(parts) == 2 {
+			processType = parts[0]
+			processIndex, err = strconv.Atoi(parts[1])
+			if err != nil {
+				return fmt.Errorf("Error parsing process index: %w", err)
+			}
+		}
+		labelSelector = append(labelSelector, fmt.Sprintf("dokku.com/process-type=%s", processType))
+	}
+
+	podList, err := clientset.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: strings.Join(labelSelector, ","),
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing pods: %w", err)
+	}
+
+	if podList.Items == nil {
+		return fmt.Errorf("No pods found for app %s", appName)
+	}
+	if len(podList.Items) == 0 {
+		return fmt.Errorf("No pods found for app %s", appName)
+	}
+
+	pods := []string{}
+	for _, pod := range podList.Items {
+		pods = append(pods, pod.Name)
+	}
+	slices.Sort(pods)
+
+	processIndex--
+	if processIndex > len(podList.Items) {
+		return fmt.Errorf("Process index %d out of range for app %s", processIndex, appName)
+	}
+
+	var selectedPod corev1.Pod
+	if podName != "" {
+		for _, pod := range podList.Items {
+			if pod.Name == podName {
+				selectedPod = pod
+				break
+			}
+		}
+	} else {
+		selectedPod = podList.Items[processIndex]
+	}
+
+	processType, ok := selectedPod.Labels["dokku.com/process-type"]
+	if !ok {
+		return fmt.Errorf("Pod %s does not have a process type label", selectedPod.Name)
+	}
+
+	command := args
+	if len(args) == 0 {
+		command = []string{"/bin/bash"}
+		if globalShell, err := common.PlugnTriggerOutputAsString("config-get-global", []string{"DOKKU_APP_SHELL"}...); err == nil && globalShell != "" {
+			command = []string{globalShell}
+		}
+		if appShell, err := common.PlugnTriggerOutputAsString("config-get", []string{appName, "DOKKU_APP_SHELL"}...); err == nil && appShell != "" {
+			command = []string{appShell}
+		}
+	}
+
+	coreclient, err := corev1client.NewForConfig(&clientset.RestConfig)
+	if err != nil {
+		return fmt.Errorf("Error creating corev1 client: %w", err)
+	}
+
+	req := coreclient.RESTClient().Post().
+		Resource("pods").
+		Namespace(selectedPod.Namespace).
+		Name(selectedPod.Name).
+		SubResource("exec")
+
+	req.Param("container", fmt.Sprintf("%s-%s", appName, processType))
+	req.Param("stdin", "true")
+	req.Param("stdout", "true")
+	req.Param("stderr", "true")
+	req.Param("tty", "true")
+
+	if selectedPod.Annotations["dokku.com/builder-type"] == "herokuish" {
+		req.Param("command", "/exec")
+	}
+	for _, cmd := range command {
+		req.Param("command", cmd)
+	}
+
+	t := term.TTY{
+		In:  os.Stdin,
+		Out: os.Stdout,
+		Raw: true,
+	}
+	size := t.GetSize()
+	sizeQueue := t.MonitorSize(size)
+
+	return t.Safe(func() error {
+		exec, err := remotecommand.NewSPDYExecutor(&clientset.RestConfig, "POST", req.URL())
+		if err != nil {
+			return fmt.Errorf("Error creating executor: %w", err)
+		}
+
+		return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+			Stdin:             os.Stdin,
+			Stdout:            os.Stdout,
+			Stderr:            os.Stderr,
+			Tty:               true,
+			TerminalSizeQueue: sizeQueue,
+		})
+	})
+}

From 55d7a4a1a0a5ffe734458ceeb0be9564b318309d Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 20:59:42 -0500
Subject: [PATCH 012/113] docs: document that enter is implemented

---
 docs/deployment/schedulers/k3s.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index bfb87feca..3da7119c9 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -156,6 +156,7 @@ The following sections describe implemented and unimplemented scheduler function
 This plugin implements various functionality through `plugn` triggers to integrate with Docker for running apps on a single server. The following functionality is supported by the `scheduler-docker-local` plugin.
 
 - `apps:destroy`
+- `enter`
 - `deploy`
 - `logs`
 - `ps:stop`
@@ -165,7 +166,6 @@ This plugin implements various functionality through `plugn` triggers to integra
 - `apps:clone`
 - `apps:rename`
 - `ps:inspect`
-- `enter`
 - `run`
 - `run:detached`
 

From c2d1f80bc47be2137f3f62c9afcbd820ffc82900 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 20:59:51 -0500
Subject: [PATCH 013/113] feat: implement app deletion

---
 plugins/scheduler-k3s/triggers.go | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index c753d6b61..f40807ef4 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -60,6 +60,17 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
 
 // TriggerPostDelete destroys the scheduler-k3s data for a given app container
 func TriggerPostDelete(appName string) error {
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("Error creating helm agent: %w", err)
+	}
+
+	err = helmAgent.UninstallChart(fmt.Sprintf("dokku-%s", appName))
+	if err != nil {
+		return fmt.Errorf("Error uninstalling chart: %w", err)
+	}
+
 	dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName)
 	propertyErr := common.PropertyDestroy("scheduler-k3s", appName)
 

From 9c891563c3ab7fbfff7c8f9c60816a4f16543d4b Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 21:48:06 -0500
Subject: [PATCH 014/113] feat: implement logs command for kubernetes

---
 plugins/scheduler-k3s/Makefile                |   2 +-
 .../scheduler-k3s/src/triggers/triggers.go    |  23 ++++
 plugins/scheduler-k3s/triggers.go             | 112 ++++++++++++++++++
 3 files changed, 136 insertions(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index b12cc544a..7efff411e 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index f5095470e..4ac250d8f 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	flag "github.com/spf13/pflag"
@@ -59,6 +60,28 @@ func main() {
 		}
 
 		err = scheduler_k3s.TriggerSchedulerEnter(scheduler, appName, containerType, ptr.Deref(podIdentifier, ""), args)
+	case "scheduler-logs":
+		var tail bool
+		var quiet bool
+		var numLines int64
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		processType := flag.Arg(2)
+		tail, err = strconv.ParseBool(flag.Arg(3))
+		if err != nil {
+			tail = false
+		}
+		quiet, err = strconv.ParseBool(flag.Arg(4))
+		if err != nil {
+			quiet = false
+		}
+
+		numLines, err = strconv.ParseInt(flag.Arg(5), 10, 64)
+		if err != nil {
+			numLines = 0
+		}
+
+		err = scheduler_k3s.TriggerSchedulerLogs(scheduler, appName, processType, tail, quiet, numLines)
 	default:
 		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
 	}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index f40807ef4..e6b0a8131 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -1,9 +1,11 @@
 package scheduler_k3s
 
 import (
+	"bufio"
 	"context"
 	"encoding/base64"
 	"fmt"
+	"io"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -15,15 +17,18 @@ import (
 
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/dokku/dokku/plugins/config"
+	"github.com/fatih/color"
 	"github.com/rancher/wharfie/pkg/registries"
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/kubectl/pkg/util/term"
+	"k8s.io/utils/ptr"
 )
 
 // TriggerInstall runs the install step for the scheduler-k3s plugin
@@ -513,3 +518,110 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		})
 	})
 }
+
+// TriggerSchedulerLogs displays logs for a given application
+func TriggerSchedulerLogs(scheduler string, appName string, processType string, tail bool, quiet bool, numLines int64) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	cSignal := make(chan os.Signal, 2)
+	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-cSignal
+		cancel()
+	}()
+
+	labelSelector := []string{fmt.Sprintf("dokku.com/app-name=%s", appName)}
+	processIndex := 0
+	if processType != "" {
+		parts := strings.SplitN(processType, ".", 2)
+		if len(parts) == 2 {
+			processType = parts[0]
+			processIndex, err = strconv.Atoi(parts[1])
+			if err != nil {
+				return fmt.Errorf("Error parsing process index: %w", err)
+			}
+		}
+		labelSelector = append(labelSelector, fmt.Sprintf("dokku.com/process-type=%s", processType))
+	}
+
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	podList, err := clientset.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: strings.Join(labelSelector, ","),
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing pods: %w", err)
+	}
+	if len(podList.Items) == 0 {
+		return fmt.Errorf("No pods found for app %s", appName)
+	}
+
+	ch := make(chan bool)
+	podItems := podList.Items
+
+	colors := []color.Attribute{
+		color.FgRed,
+		color.FgYellow,
+		color.FgGreen,
+		color.FgCyan,
+		color.FgBlue,
+		color.FgMagenta,
+	}
+	// colorIndex := 0
+	for i := 0; i < len(podItems); i++ {
+		if processIndex > 0 && i != (processIndex-1) {
+			continue
+		}
+
+		logOptions := v1.PodLogOptions{
+			Follow: tail,
+		}
+		if numLines > 0 {
+			logOptions.TailLines = ptr.To(numLines)
+		}
+
+		podColor := colors[i%len(colors)]
+		dynoText := color.New(podColor).SprintFunc()
+		podName := podItems[i].Name
+		podLogs, err := clientset.Client.CoreV1().Pods(namespace).GetLogs(podName, &logOptions).Stream(ctx)
+		if err != nil {
+			return err
+		}
+		buffer := bufio.NewReader(podLogs)
+		go func(buffer *bufio.Reader, prettyText func(a ...interface{}) string, ch chan bool) {
+			defer func() {
+				ch <- true
+			}()
+			for {
+				str, readErr := buffer.ReadString('\n')
+				if readErr == io.EOF {
+					break
+				}
+
+				if str == "" {
+					continue
+				}
+
+				if !quiet {
+					str = fmt.Sprintf("%s %s", dynoText(fmt.Sprintf("app[%s]:", podName)), str)
+				}
+
+				_, err := fmt.Print(str)
+				if err != nil {
+					return
+				}
+			}
+		}(buffer, dynoText, ch)
+	}
+	<-ch
+
+	return nil
+}

From 57b62f51d352a2c38f75a7737c639ad114817b31 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 23:26:57 -0500
Subject: [PATCH 015/113] refactor: update log streaming so plugin triggers
 don't get abandoned on ctrl-c

We now propagate a cancellable context object on a new plugin trigger call that allows for signals to work as expected.
---
 plugins/logs/subcommands.go  | 10 ++++++----
 plugins/scheduler-k3s/go.mod | 15 ++++++++-------
 plugins/scheduler-k3s/go.sum | 25 ++++++++++++++++---------
 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/plugins/logs/subcommands.go b/plugins/logs/subcommands.go
index 775bb1b71..bbf9318da 100644
--- a/plugins/logs/subcommands.go
+++ b/plugins/logs/subcommands.go
@@ -25,10 +25,12 @@ func CommandDefault(appName string, num int64, process string, tail, quiet bool)
 	q := strconv.FormatBool(quiet)
 	n := strconv.FormatInt(num, 10)
 
-	if err := common.PlugnTrigger("scheduler-logs", []string{s, appName, process, t, q, n}...); err != nil {
-		return err
-	}
-	return nil
+	_, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Args:        []string{s, appName, process, t, q, n},
+		StreamStdio: true,
+		Trigger:     "scheduler-logs",
+	})
+	return err
 }
 
 // CommandFailed shows the last failed deploy logs
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 0fb4be71f..1631b78cd 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -5,6 +5,7 @@ go 1.21
 require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa
+	github.com/fatih/color v1.16.0
 	github.com/go-resty/resty/v2 v2.11.0
 	github.com/rancher/wharfie v0.6.5
 	github.com/spf13/pflag v1.0.5
@@ -16,7 +17,6 @@ require (
 	k8s.io/apimachinery v0.29.0
 	k8s.io/cli-runtime v0.29.0
 	k8s.io/client-go v0.29.0
-	k8s.io/klog/v2 v2.110.1
 	k8s.io/kubectl v0.28.4
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	mvdan.cc/sh/v3 v3.7.0
@@ -32,6 +32,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Microsoft/hcsshim v0.11.0 // indirect
+	github.com/alexellis/go-execute/v2 v2.2.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -54,7 +55,6 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
-	github.com/fatih/color v1.15.0 // indirect
 	github.com/go-acme/lego/v4 v4.14.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
@@ -137,14 +137,14 @@ require (
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.16.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
@@ -156,6 +156,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index 67ba25a3a..fa4442fb9 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -27,6 +27,8 @@ github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/O
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alexellis/go-execute/v2 v2.2.1 h1:4Ye3jiCKQarstODOEmqDSRCqxMHLkC92Bhse743RdOI=
+github.com/alexellis/go-execute/v2 v2.2.1/go.mod h1:FMdRnUTiFAmYXcv23txrp3VYZfLo24nMpiIneWgKHTQ=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
@@ -110,8 +112,8 @@ github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCv
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
@@ -322,8 +324,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
-github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
-github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE=
+github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
@@ -451,8 +453,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -479,8 +482,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
 golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
@@ -512,16 +516,18 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -529,8 +535,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From 3137c457faf9884aac0c324d47ea4d2b42a2da9d Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 23:28:10 -0500
Subject: [PATCH 016/113] fix: kill log tailing properly on ctrl+c

---
 plugins/scheduler-k3s/triggers.go | 69 ++++++++++++++++++-------------
 1 file changed, 41 insertions(+), 28 deletions(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index e6b0a8131..76a4e3a5c 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -392,13 +392,14 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	cSignal := make(chan os.Signal, 2)
-	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
 	go func() {
-		<-cSignal
-		common.LogWarn("Exiting pod")
+		<-signals
 		cancel()
 	}()
 
@@ -530,12 +531,14 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	cSignal := make(chan os.Signal, 2)
-	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
 	go func() {
-		<-cSignal
+		<-signals
 		cancel()
 	}()
 
@@ -567,6 +570,10 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 	ch := make(chan bool)
 	podItems := podList.Items
 
+	if os.Getenv("FORCE_COLOR") == "1" {
+		color.NoColor = false
+	}
+
 	colors := []color.Attribute{
 		color.FgRed,
 		color.FgYellow,
@@ -596,30 +603,36 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 			return err
 		}
 		buffer := bufio.NewReader(podLogs)
-		go func(buffer *bufio.Reader, prettyText func(a ...interface{}) string, ch chan bool) {
+		go func(ctx context.Context, buffer *bufio.Reader, prettyText func(a ...interface{}) string, ch chan bool) {
 			defer func() {
 				ch <- true
 			}()
 			for {
-				str, readErr := buffer.ReadString('\n')
-				if readErr == io.EOF {
-					break
-				}
-
-				if str == "" {
-					continue
-				}
-
-				if !quiet {
-					str = fmt.Sprintf("%s %s", dynoText(fmt.Sprintf("app[%s]:", podName)), str)
-				}
-
-				_, err := fmt.Print(str)
-				if err != nil {
+				select {
+				case <-ctx.Done(): // if cancel() execute
+					ch <- true
 					return
+				default:
+					str, readErr := buffer.ReadString('\n')
+					if readErr == io.EOF {
+						break
+					}
+
+					if str == "" {
+						continue
+					}
+
+					if !quiet {
+						str = fmt.Sprintf("%s %s", dynoText(fmt.Sprintf("app[%s]:", podName)), str)
+					}
+
+					_, err := fmt.Print(str)
+					if err != nil {
+						return
+					}
 				}
 			}
-		}(buffer, dynoText, ch)
+		}(ctx, buffer, dynoText, ch)
 	}
 	<-ch
 

From 8a719a28809cc3982ea4b1783baaf1e93d949a49 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 17 Jan 2024 23:28:21 -0500
Subject: [PATCH 017/113] docs: update docs to point out unsupported
 functionality

---
 docs/deployment/schedulers/k3s.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 3da7119c9..aa71773b1 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -175,10 +175,11 @@ The following Dokku functionality is not implemented at this time.
 - `vector` log integration
 - one-off tasks
 - persistent storage
+- healthchecks
 
 ### Logging support
 
-App logs for the `logs` command are fetched by Dokku from running containers via the `kubectl` cli. Persisting logs via Vector is not implemented at this time.
+App logs for the `logs` command are fetched by Dokku from running containers via the `kubectl` cli. Persisting logs via Vector is not implemented at this time. Users may choose to configure the Vector Kubernetes integration directly by following [this guide](https://vector.dev/docs/setup/installation/platforms/kubernetes/).
 
 ### Supported Resource Management Properties
 

From 46b6f446fdd62ac095f1a3eaab3bc693b6b5e9a7 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 00:19:20 -0500
Subject: [PATCH 018/113] feat: implement ps:stop for k3s scheduler

This simply scales down processes.
---
 plugins/scheduler-k3s/Makefile                |  2 +-
 .../scheduler-k3s/src/triggers/triggers.go    |  4 ++
 plugins/scheduler-k3s/triggers.go             | 54 +++++++++++++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 7efff411e..8609e8ee3 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index 4ac250d8f..7b8e74f0f 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -82,6 +82,10 @@ func main() {
 		}
 
 		err = scheduler_k3s.TriggerSchedulerLogs(scheduler, appName, processType, tail, quiet, numLines)
+	case "scheduler-stop":
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		err = scheduler_k3s.TriggerSchedulerStop(scheduler, appName)
 	default:
 		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
 	}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 76a4e3a5c..9f27fb55d 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -22,6 +22,7 @@ import (
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
+	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -638,3 +639,56 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 
 	return nil
 }
+
+// TriggerSchedulerStop stops an application
+func TriggerSchedulerStop(scheduler string, appName string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	deploymentList, err := clientset.Client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing deployments: %w", err)
+	}
+
+	for _, deployment := range deploymentList.Items {
+		processType, ok := deployment.Annotations["dokku.com/process-type"]
+		if !ok {
+			return fmt.Errorf("Deployment %s does not have a process type annotation", deployment.Name)
+		}
+		common.LogVerboseQuiet(fmt.Sprintf("Stopping %s process", processType))
+		_, err := clientset.Client.AppsV1().Deployments(namespace).UpdateScale(ctx, deployment.Name, &autoscalingv1.Scale{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      deployment.Name,
+				Namespace: namespace,
+			},
+			Spec: autoscalingv1.ScaleSpec{
+				Replicas: 0,
+			},
+		}, metav1.UpdateOptions{})
+		if err != nil {
+			return fmt.Errorf("Error updating deployment scale: %w", err)
+		}
+	}
+
+	return nil
+}

From dfa7d0f78c78d0a14ffd167909670b78cec3e4fc Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 00:24:50 -0500
Subject: [PATCH 019/113] feat: mark created namespaces as managed by dokku

---
 plugins/scheduler-k3s/functions.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index c8ec555e4..314a6d6d2 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -101,6 +101,12 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 	namespace := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: namespaceName,
+			Annotations: map[string]string{
+				"dokku.com/managed": "true",
+			},
+			Labels: map[string]string{
+				"dokku.com/managed": "true",
+			},
 		},
 	}
 	_, err = clientset.Client.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{})

From 2911218816049e704725f44878688684aa6a9563 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 01:02:16 -0500
Subject: [PATCH 020/113] feat: add an annotations everywhere to ensure folks
 know what is managed by dokku

---
 plugins/scheduler-k3s/template.go | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 154baafb1..33f961928 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -201,14 +201,15 @@ func printResource(input PrintInput) error {
 func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	labels := map[string]string{
 		"dokku.com/app-name":         input.AppName,
-		"dokku.com/process-type":     input.ProcessType,
 		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"dokku.com/process-type":     input.ProcessType,
 	}
 	annotations := map[string]string{
 		"dokku.com/app-name":      input.AppName,
-		"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
-		"dokku.com/process-type":  input.ProcessType,
 		"dokku.com/builder-type":  input.ImageSourceType,
+		"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
+		"dokku.com/managed":       "true",
+		"dokku.com/process-type":  input.ProcessType,
 	}
 	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
 
@@ -346,8 +347,12 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 			Name:      fmt.Sprintf("%s-%s", input.ServiceName, port),
 			Namespace: input.Namespace,
 			Labels: map[string]string{
-				"dokku.com/app-name":     input.AppName,
-				"dokku.com/process-type": input.ProcessType,
+				"dokku.com/app-name":         input.AppName,
+				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+				"dokku.com/process-type":     input.ProcessType,
+			},
+			Annotations: map[string]string{
+				"dokku.com/managed": "true",
 			},
 		},
 		Spec: traefikv1alpha1.IngressRouteSpec{
@@ -389,6 +394,9 @@ func templateKubernetesSecret(input Secret) corev1.Secret {
 				"dokku.com/app-name":      input.AppName,
 				"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
 			},
+			Annotations: map[string]string{
+				"dokku.com/managed": "true",
+			},
 		},
 		Data: map[string][]byte{},
 	}
@@ -403,15 +411,18 @@ func templateKubernetesService(input Service) corev1.Service {
 			Namespace: input.Namespace,
 			Labels: map[string]string{
 				"dokku.com/app-name":         input.AppName,
-				"dokku.com/process-type":     "web",
 				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
+				"dokku.com/process-type":     "web",
+			},
+			Annotations: map[string]string{
+				"dokku.com/managed": "true",
 			},
 		},
 		Spec: corev1.ServiceSpec{
 			Selector: map[string]string{
 				"dokku.com/app-name":         input.AppName,
-				"dokku.com/process-type":     "web",
 				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
+				"dokku.com/process-type":     "web",
 			},
 			Ports: []corev1.ServicePort{
 				{

From 1e11ec3911d1e42b2b8d55ffcf4d08b0cfacb6da Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 01:02:49 -0500
Subject: [PATCH 021/113] fix: setup all port maps correctly on service

---
 plugins/scheduler-k3s/template.go | 25 +++++++++++++++----------
 plugins/scheduler-k3s/triggers.go |  2 +-
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 33f961928..b1789a972 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -62,7 +62,6 @@ type IngressRoute struct {
 	Entrypoints []IngressRouteEntrypoint
 	Hostnames   []string
 	Namespace   string
-	Port        int32
 	PortMap     PortMap
 	ProcessType string
 	ServiceName string
@@ -77,7 +76,7 @@ type Secret struct {
 type Service struct {
 	AppName   string
 	Namespace string
-	Port      int32
+	PortMaps  []PortMap
 }
 
 type PrintInput struct {
@@ -129,7 +128,6 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 				AppName:     input.AppName,
 				Hostnames:   domains,
 				Namespace:   input.Namespace,
-				Port:        input.Deployment.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort,
 				PortMap:     portMap,
 				ProcessType: "web",
 				ServiceName: input.Service.Name,
@@ -424,15 +422,22 @@ func templateKubernetesService(input Service) corev1.Service {
 				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
 				"dokku.com/process-type":     "web",
 			},
-			Ports: []corev1.ServicePort{
-				{
-					Name:       "web",
-					Port:       input.Port,
-					TargetPort: intstr.FromInt32(input.Port),
-				},
-			},
 		},
 	}
+
+	for _, portMap := range input.PortMaps {
+		protocol := "TCP"
+		if portMap.Scheme == "udp" {
+			protocol = "UDP"
+		}
+		service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{
+			Name:       fmt.Sprintf("%s-%d-%d", portMap.Scheme, portMap.HostPort, portMap.ContainerPort),
+			Port:       portMap.HostPort,
+			TargetPort: intstr.FromString(fmt.Sprintf("%s-%d-%d", portMap.Scheme, portMap.HostPort, portMap.ContainerPort)),
+			Protocol:   corev1.Protocol(protocol),
+		})
+	}
+
 	return service
 }
 
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 9f27fb55d..cc26c832e 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -289,7 +289,7 @@ data:
 		service := templateKubernetesService(Service{
 			AppName:   appName,
 			Namespace: namespace,
-			Port:      deployment.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort,
+			PortMaps:  portMaps,
 		})
 
 		err := printResource(PrintInput{

From a2504fadd2469e7bade033ec69906359314c6fc8 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 01:04:23 -0500
Subject: [PATCH 022/113] chore: add a note about stopping nginx

---
 docs/deployment/schedulers/k3s.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index aa71773b1..21ead7c48 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -75,6 +75,12 @@ As routing is handled by traefik managed on the k3s plugin, set the proxy plugin
 dokku proxy:set --global k3s
 ```
 
+Ensure any other proxy implementations are disabled. Typically at least nginx will be running on the Dokku host and should be stopped if the host is used as load balancer.
+
+```shell
+dokku nginx:stop
+```
+
 Finally, set the scheduler to `k3s` so that app deploys will work on k3s.
 
 ```shell

From 6b85374857238bfff01bb74408da2079c8322d11 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 01:08:02 -0500
Subject: [PATCH 023/113] feat: implement scheduler-k3s:show-config

---
 plugins/scheduler-k3s/Makefile                   |  2 +-
 .../scheduler-k3s/src/subcommands/subcommands.go |  4 ++++
 plugins/scheduler-k3s/subcommands.go             | 16 ++++++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 8609e8ee3..fe33daf90 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set
+SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set subcommands/show-kubeconfig
 TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go
index bc6bd03ab..7915d1fe7 100644
--- a/plugins/scheduler-k3s/src/subcommands/subcommands.go
+++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go
@@ -44,6 +44,10 @@ func main() {
 			value = args.Arg(1)
 		}
 		err = scheduler_k3s.CommandSet(appName, property, value)
+	case "show-kubeconfig":
+		args := flag.NewFlagSet("scheduler-k3s:show-kubeconfig", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		err = scheduler_k3s.CommandShowKubeconfig()
 	default:
 		err = fmt.Errorf("Invalid plugin subcommand call: %s", subcommand)
 	}
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index ed28374a3..e149d6a87 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -116,3 +116,19 @@ func CommandSet(appName string, property string, value string) error {
 	common.CommandPropertySet("scheduler-k3s", appName, property, value, DefaultProperties, GlobalProperties)
 	return nil
 }
+
+// CommandShowKubeconfig displays the kubeconfig file contents
+func CommandShowKubeconfig() error {
+	if !common.FileExists(KubeConfigPath) {
+		return fmt.Errorf("Kubeconfig file does not exist: %s", KubeConfigPath)
+	}
+
+	b, err := os.ReadFile(KubeConfigPath)
+	if err != nil {
+		return fmt.Errorf("Unable to read kubeconfig file: %w", err)
+	}
+
+	fmt.Println(string(b))
+
+	return nil
+}

From 8b8444fb1f1a2dcb805399471a6267c898858dfb Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 05:28:07 -0500
Subject: [PATCH 024/113] feat: implement run and run:detached

This utilizes a Kubernetes Job to start a pod in order to take advantage of completed Job cleanup in Kubernetes. It also properly handles interactivity by ensuring the container has a TTY.

Note that there is currently no provision for parsing docker arguments and translating them into their Kubernetes equivalents.
---
 docs/deployment/schedulers/k3s.md             |   8 +-
 plugins/scheduler-k3s/Makefile                |   2 +-
 plugins/scheduler-k3s/functions.go            |  19 +-
 plugins/scheduler-k3s/go.mod                  |  13 +-
 plugins/scheduler-k3s/go.sum                  |  30 +-
 plugins/scheduler-k3s/helm.go                 |  93 +++-
 plugins/scheduler-k3s/scheduler_k3s.go        |   2 +
 .../scheduler-k3s/src/triggers/triggers.go    |  21 +-
 plugins/scheduler-k3s/template.go             | 266 ++++++++--
 plugins/scheduler-k3s/triggers.go             | 494 +++++++++++++++++-
 10 files changed, 833 insertions(+), 115 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 21ead7c48..ced87c825 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -162,6 +162,9 @@ The following sections describe implemented and unimplemented scheduler function
 This plugin implements various functionality through `plugn` triggers to integrate with Docker for running apps on a single server. The following functionality is supported by the `scheduler-docker-local` plugin.
 
 - `apps:destroy`
+- `run`
+       - The `scheduler-post-run` trigger is not always triggered
+- `run:detached`
 - `enter`
 - `deploy`
 - `logs`
@@ -171,15 +174,14 @@ This plugin implements various functionality through `plugn` triggers to integra
 
 - `apps:clone`
 - `apps:rename`
+- `run:list`
+- `run:logs`
 - `ps:inspect`
-- `run`
-- `run:detached`
 
 The following Dokku functionality is not implemented at this time.
 
 - `cron`
 - `vector` log integration
-- one-off tasks
 - persistent storage
 - healthchecks
 
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index fe33daf90..3ea295207 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set subcommands/show-kubeconfig
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-stop
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-run triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 314a6d6d2..d4fb376ac 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/dokku/dokku/plugins/config"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -81,6 +82,20 @@ func KubernetesClientConfig() clientcmd.ClientConfig {
 		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
 }
 
+func createKubernetesJob(ctx context.Context, job batchv1.Job) (batchv1.Job, error) {
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return batchv1.Job{}, err
+	}
+
+	createdJob, err := clientset.Client.BatchV1().Jobs(job.Namespace).Create(ctx, &job, metav1.CreateOptions{})
+	if err != nil {
+		return batchv1.Job{}, err
+	}
+
+	return *createdJob, nil
+}
+
 func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
 	clientset, err := NewKubernetesClient()
 	if err != nil {
@@ -98,7 +113,7 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 		}
 	}
 
-	namespace := &corev1.Namespace{
+	namespace := corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: namespaceName,
 			Annotations: map[string]string{
@@ -109,7 +124,7 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 			},
 		},
 	}
-	_, err = clientset.Client.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{})
+	_, err = clientset.Client.CoreV1().Namespaces().Create(ctx, &namespace, metav1.CreateOptions{})
 	if err != nil {
 		return err
 	}
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 1631b78cd..e5912efa4 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa
 	github.com/fatih/color v1.16.0
 	github.com/go-resty/resty/v2 v2.11.0
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/rancher/wharfie v0.6.5
 	github.com/spf13/pflag v1.0.5
 	github.com/traefik/traefik/v2 v2.10.7
@@ -18,6 +19,7 @@ require (
 	k8s.io/cli-runtime v0.29.0
 	k8s.io/client-go v0.29.0
 	k8s.io/kubectl v0.28.4
+	k8s.io/kubernetes v1.29.1
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	mvdan.cc/sh/v3 v3.7.0
 )
@@ -133,12 +135,13 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
-	go.opentelemetry.io/otel/trace v1.14.0 // indirect
+	go.opentelemetry.io/otel v1.19.0 // indirect
+	go.opentelemetry.io/otel/metric v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.16.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
@@ -146,9 +149,9 @@ require (
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
+	golang.org/x/tools v0.16.1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
 	google.golang.org/grpc v1.58.3 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index fa4442fb9..79ff56cb4 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -241,8 +241,10 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
-github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
+github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
@@ -438,10 +440,12 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -464,8 +468,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -550,8 +554,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E=
-golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA=
+golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -563,8 +567,8 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -624,6 +628,8 @@ k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/A
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/kubectl v0.28.4 h1:gWpUXW/T7aFne+rchYeHkyB8eVDl5UZce8G4X//kjUQ=
 k8s.io/kubectl v0.28.4/go.mod h1:CKOccVx3l+3MmDbkXtIUtibq93nN2hkDR99XDCn7c/c=
+k8s.io/kubernetes v1.29.1 h1:fxJFVb8uqbYZDYHpwIsAndBQs360cQGb0xa1gYFh3fo=
+k8s.io/kubernetes v1.29.1/go.mod h1:xZPKU0yO0CBbLTnbd+XGyRmmtmaVuJykDb8gNCkeeUE=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 mvdan.cc/sh/v3 v3.7.0 h1:lSTjdP/1xsddtaKfGg7Myu7DnlHItd3/M2tomOcNNBg=
diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go
index e5f462d6c..b846bf154 100644
--- a/plugins/scheduler-k3s/helm.go
+++ b/plugins/scheduler-k3s/helm.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"sort"
 	"strings"
 	"syscall"
 	"time"
@@ -16,6 +17,7 @@ import (
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/kube"
 	"helm.sh/helm/v3/pkg/storage/driver"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var DevNullPrinter = func(format string, v ...interface{}) {}
@@ -38,6 +40,21 @@ var DeployLogPrinter = func(format string, v ...interface{}) {
 	}
 }
 
+type ChartInput struct {
+	ChartPath         string
+	Namespace         string
+	ReleaseName       string
+	RollbackOnFailure bool
+	Timeout           time.Duration
+	Values            map[string]interface{}
+}
+
+type Release struct {
+	Name      string
+	Namespace string
+	Version   int
+}
+
 type HelmAgent struct {
 	Configuration *action.Configuration
 	Namespace     string
@@ -45,18 +62,6 @@ type HelmAgent struct {
 }
 
 func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error) {
-	actionConfig, err := getActionConfig(namespace, logger)
-	if err != nil {
-		return nil, err
-	}
-
-	return &HelmAgent{
-		Configuration: actionConfig,
-		Namespace:     namespace,
-	}, nil
-}
-
-func getActionConfig(namespace string, logger action.DebugLog) (*action.Configuration, error) {
 	actionConfig := new(action.Configuration)
 
 	helmDriver := os.Getenv("HELM_DRIVER")
@@ -68,7 +73,12 @@ func getActionConfig(namespace string, logger action.DebugLog) (*action.Configur
 	if err := actionConfig.Init(kubeConfig, namespace, helmDriver, logger); err != nil {
 		return nil, err
 	}
-	return actionConfig, nil
+
+	return &HelmAgent{
+		Configuration: actionConfig,
+		Namespace:     namespace,
+		Logger:        logger,
+	}, nil
 }
 
 func (h *HelmAgent) ChartExists(releaseName string) (bool, error) {
@@ -89,19 +99,35 @@ func (h *HelmAgent) ChartExists(releaseName string) (bool, error) {
 	return false, nil
 }
 
-type ChartInput struct {
-	ChartPath         string
-	Namespace         string
-	ReleaseName       string
-	RollbackOnFailure bool
-	Timeout           time.Duration
-	Values            map[string]interface{}
+func (h *HelmAgent) DeleteRevision(releaseName string, revision int) error {
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	secretName := fmt.Sprintf("sh.helm.release.v1.%s.v%d", releaseName, revision)
+	err = clientset.Client.CoreV1().Secrets(h.Namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("Error deleting secret: %w", err)
+	}
+	return nil
+}
+
+func (h *HelmAgent) GetValues(releaseName string) (map[string]interface{}, error) {
+	client := action.NewGetValues(h.Configuration)
+	client.AllValues = true
+	values, err := client.Run(releaseName)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting values: %w", err)
+	}
+
+	return values, nil
 }
 
 func (h *HelmAgent) InstallOrUpgradeChart(input ChartInput) error {
 	chartExists, err := h.ChartExists(input.ReleaseName)
 	if err != nil {
-		return err
+		return fmt.Errorf("Error checking if chart exists: %w", err)
 	}
 
 	if chartExists {
@@ -167,6 +193,29 @@ func (h *HelmAgent) InstallChart(input ChartInput) error {
 	return nil
 }
 
+func (h *HelmAgent) ListRevisions(releaseName string) ([]Release, error) {
+	client := action.NewHistory(h.Configuration)
+	response, err := client.Run(releaseName)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting revisions: %w", err)
+	}
+
+	releases := []Release{}
+	for _, release := range response {
+		releases = append(releases, Release{
+			Name:      release.Name,
+			Namespace: release.Namespace,
+			Version:   release.Version,
+		})
+	}
+
+	sort.Slice(releases, func(i, j int) bool {
+		return releases[i].Version < releases[j].Version
+	})
+
+	return releases, nil
+}
+
 func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 	namespace := input.Namespace
 	if namespace == "" {
@@ -186,7 +235,9 @@ func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 	client := action.NewUpgrade(h.Configuration)
 	client.Atomic = input.RollbackOnFailure
 	client.ChartPathOptions = action.ChartPathOptions{}
+	client.CleanupOnFail = true
 	client.DryRun = false
+	client.MaxHistory = 10
 	client.Namespace = namespace
 	client.Timeout = input.Timeout
 	client.Wait = false
diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 6837b91fe..02462ad14 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -5,6 +5,7 @@ import (
 
 	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -43,6 +44,7 @@ var k8sNativeSchemeOnce sync.Once
 func init() {
 	k8sNativeSchemeOnce.Do(func() {
 		_ = appsv1.AddToScheme(runtimeScheme)
+		_ = batchv1.AddToScheme(runtimeScheme)
 		_ = corev1.AddToScheme(runtimeScheme)
 		_ = traefikv1alpha1.AddToScheme(runtimeScheme)
 	})
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index 7b8e74f0f..392dbb0e6 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -53,7 +53,10 @@ func main() {
 		appName := flag.Arg(1)
 		containerType := flag.Arg(2)
 		args := flag.Args()
-		if len(args) > 3 {
+		if len(args) == 2 {
+			_, args = common.ShiftString(args)
+			_, args = common.ShiftString(args)
+		} else if len(args) >= 3 {
 			_, args = common.ShiftString(args)
 			_, args = common.ShiftString(args)
 			_, args = common.ShiftString(args)
@@ -86,6 +89,22 @@ func main() {
 		scheduler := flag.Arg(0)
 		appName := flag.Arg(1)
 		err = scheduler_k3s.TriggerSchedulerStop(scheduler, appName)
+	case "scheduler-run":
+		var envCount int
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		envCount, err = strconv.Atoi(flag.Arg(2))
+		if err != nil {
+			envCount = 0
+		}
+		args := flag.Args()
+		if len(args) >= 3 {
+			_, args = common.ShiftString(args)
+			_, args = common.ShiftString(args)
+			_, args = common.ShiftString(args)
+		}
+
+		err = scheduler_k3s.TriggerSchedulerRun(scheduler, appName, envCount, args)
 	default:
 		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
 	}
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index b1789a972..53c582ac4 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -11,6 +11,7 @@ import (
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -48,6 +49,7 @@ type Deployment struct {
 	PortMaps         []PortMap
 	ProcessType      string
 	Replicas         int32
+	WorkingDir       string
 }
 
 type IngressRouteEntrypoint string
@@ -67,6 +69,23 @@ type IngressRoute struct {
 	ServiceName string
 }
 
+type Job struct {
+	AppName          string
+	Command          []string
+	DeploymentID     int64
+	Entrypoint       string
+	Env              map[string]string
+	Image            string
+	ImagePullSecrets string
+	ImageSourceType  string
+	Interactive      bool
+	Labels           map[string]string
+	Namespace        string
+	ProcessType      string
+	RemoveContainer  bool
+	WorkingDir       string
+}
+
 type Secret struct {
 	AppName   string
 	Env       map[string]string
@@ -79,11 +98,10 @@ type Service struct {
 	PortMaps  []PortMap
 }
 
-type PrintInput struct {
+type WriteResourceInput struct {
 	AppendContents string
 	Object         runtime.Object
 	Path           string
-	Name           string
 	Replacements   *orderedmap.OrderedMap[string, string]
 }
 
@@ -133,10 +151,9 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 				ServiceName: input.Service.Name,
 			})
 
-			err = printResource(PrintInput{
+			err = writeResourceToFile(WriteResourceInput{
 				Object: &ingressRoute,
 				Path:   filepath.Join(input.ChartDir, fmt.Sprintf("templates/ingress-route-%s-%d-%d.yaml", portMap.Scheme, portMap.HostPort, portMap.ContainerPort)),
-				Name:   ingressRoute.Name,
 			})
 			if err != nil {
 				return fmt.Errorf("Error printing ingress route: %w", err)
@@ -146,56 +163,6 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 	return nil
 }
 
-func printResource(input PrintInput) error {
-	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
-	printr := printers.NewTypeSetter(runtimeScheme).ToPrinter(&printers.YAMLPrinter{})
-	handle, err := os.Create(input.Path)
-	if err != nil {
-		return fmt.Errorf("Error creating template file: %w", err)
-	}
-	defer handle.Close()
-
-	if err := printr.PrintObj(input.Object, handle); err != nil {
-		return fmt.Errorf("Error writing template file: %w", err)
-	}
-
-	if input.Replacements != nil {
-		b, err := os.ReadFile(input.Path)
-		if err != nil {
-			return fmt.Errorf("Error reading template file: %w", err)
-		}
-
-		contents := string(b)
-		for pair := input.Replacements.Oldest(); pair != nil; pair = pair.Next() {
-			contents = strings.ReplaceAll(string(contents), pair.Key, pair.Value)
-		}
-
-		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
-		if err != nil {
-			return fmt.Errorf("Error updating template file with replacements: %w", err)
-		}
-	}
-
-	if input.AppendContents != "" {
-		b, err := os.ReadFile(input.Path)
-		if err != nil {
-			return fmt.Errorf("Error reading template file: %w", err)
-		}
-
-		contents := string(b) + "\n" + input.AppendContents
-
-		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
-		if err != nil {
-			return fmt.Errorf("Error updating template file with replacements: %w", err)
-		}
-	}
-
-	if os.Getenv("DOKKU_TRACE") == "1" {
-		common.CatFile(input.Path)
-	}
-	return nil
-}
-
 func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	labels := map[string]string{
 		"dokku.com/app-name":         input.AppName,
@@ -245,11 +212,12 @@ func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 								},
 							},
 							Image:           input.Image,
-							ImagePullPolicy: corev1.PullPolicy("Always"),
+							ImagePullPolicy: corev1.PullAlways,
 							Resources: corev1.ResourceRequirements{
 								Limits:   corev1.ResourceList{},
 								Requests: corev1.ResourceList{},
 							},
+							WorkingDir: input.WorkingDir,
 						},
 					},
 				},
@@ -382,6 +350,144 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 	return ingressRoute
 }
 
+func templateKubernetesJob(input Job) (batchv1.Job, error) {
+	labels := map[string]string{
+		"dokku.com/app-name":         input.AppName,
+		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"dokku.com/process-type":     input.ProcessType,
+	}
+	annotations := map[string]string{
+		"dokku.com/app-name":      input.AppName,
+		"dokku.com/builder-type":  input.ImageSourceType,
+		"dokku.com/deployment-id": fmt.Sprint(input.DeploymentID),
+		"dokku.com/managed":       "true",
+		"dokku.com/process-type":  input.ProcessType,
+	}
+
+	// todo: implement a controller to delete pods on completion
+
+	for key, value := range input.Labels {
+		labels[key] = value
+	}
+	secretName := fmt.Sprintf("env-%s.%d", input.AppName, input.DeploymentID)
+
+	env := []corev1.EnvVar{}
+	for key, value := range input.Env {
+		env = append(env, corev1.EnvVar{
+			Name:  key,
+			Value: value,
+		})
+	}
+
+	job := batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: fmt.Sprintf("%s-%s-", input.AppName, input.ProcessType),
+			Namespace:    input.Namespace,
+			Labels:       labels,
+			Annotations:  annotations,
+		},
+		Spec: batchv1.JobSpec{
+			BackoffLimit: ptr.To(int32(0)),
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Args: input.Command,
+							Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+							Env:  env,
+							// todo: pull secret ref here
+							EnvFrom: []corev1.EnvFromSource{
+								{
+									SecretRef: &corev1.SecretEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: secretName,
+										},
+										Optional: ptr.To(true),
+									},
+								},
+							},
+							Image:           input.Image,
+							ImagePullPolicy: corev1.PullAlways,
+							Resources: corev1.ResourceRequirements{
+								Limits:   corev1.ResourceList{},
+								Requests: corev1.ResourceList{},
+							},
+							WorkingDir: input.WorkingDir,
+						},
+					},
+					RestartPolicy: corev1.RestartPolicyNever,
+				},
+			},
+		},
+	}
+
+	if input.Entrypoint != "" {
+		job.Spec.Template.Spec.Containers[0].Command = []string{input.Entrypoint}
+	}
+
+	if input.Interactive {
+		job.Spec.Template.Spec.Containers[0].Stdin = true
+		job.Spec.Template.Spec.Containers[0].StdinOnce = true
+		job.Spec.Template.Spec.Containers[0].TTY = true
+	}
+
+	if input.RemoveContainer {
+		job.Spec.TTLSecondsAfterFinished = ptr.To(int32(60))
+	}
+
+	if input.ImagePullSecrets != "" {
+		job.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
+			{
+				Name: input.ImagePullSecrets,
+			},
+		}
+	}
+
+	cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
+	if err != nil && cpuLimit != "" && cpuLimit != "0" {
+		cpuQuantity, err := resource.ParseQuantity(cpuLimit)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing cpu limit: %w", err)
+		}
+		job.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
+	}
+	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
+	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
+		nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
+		}
+		job.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
+	}
+	memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
+	if err != nil && memoryLimit != "" && memoryLimit != "0" {
+		memoryQuantity, err := resource.ParseQuantity(memoryLimit)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing memory limit: %w", err)
+		}
+		job.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
+	}
+
+	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
+	if err != nil && cpuRequest != "" && cpuRequest != "0" {
+		cpuQuantity, err := resource.ParseQuantity(cpuRequest)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing cpu request: %w", err)
+		}
+		job.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
+	}
+	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
+	if err != nil && memoryRequest != "" && memoryRequest != "0" {
+		memoryQuantity, err := resource.ParseQuantity(memoryRequest)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing memory request: %w", err)
+		}
+		job.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
+	}
+
+	return job, nil
+}
+
 func templateKubernetesSecret(input Secret) corev1.Secret {
 	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
 	secret := corev1.Secret{
@@ -441,6 +547,56 @@ func templateKubernetesService(input Service) corev1.Service {
 	return service
 }
 
+func writeResourceToFile(input WriteResourceInput) error {
+	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
+	printr := printers.NewTypeSetter(runtimeScheme).ToPrinter(&printers.YAMLPrinter{})
+	handle, err := os.Create(input.Path)
+	if err != nil {
+		return fmt.Errorf("Error creating template file: %w", err)
+	}
+	defer handle.Close()
+
+	if err := printr.PrintObj(input.Object, handle); err != nil {
+		return fmt.Errorf("Error writing template file: %w", err)
+	}
+
+	if input.Replacements != nil {
+		b, err := os.ReadFile(input.Path)
+		if err != nil {
+			return fmt.Errorf("Error reading template file: %w", err)
+		}
+
+		contents := string(b)
+		for pair := input.Replacements.Oldest(); pair != nil; pair = pair.Next() {
+			contents = strings.ReplaceAll(string(contents), pair.Key, pair.Value)
+		}
+
+		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
+		if err != nil {
+			return fmt.Errorf("Error updating template file with replacements: %w", err)
+		}
+	}
+
+	if input.AppendContents != "" {
+		b, err := os.ReadFile(input.Path)
+		if err != nil {
+			return fmt.Errorf("Error reading template file: %w", err)
+		}
+
+		contents := string(b) + "\n" + input.AppendContents
+
+		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
+		if err != nil {
+			return fmt.Errorf("Error updating template file with replacements: %w", err)
+		}
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(input.Path)
+	}
+	return nil
+}
+
 func writeYaml(input WriteYamlInput) error {
 	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
 	data, err := yaml.Marshal(input.Object)
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index cc26c832e..cfff5cb92 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -18,6 +19,7 @@ import (
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/dokku/dokku/plugins/config"
 	"github.com/fatih/color"
+	"github.com/kballard/go-shellquote"
 	"github.com/rancher/wharfie/pkg/registries"
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
@@ -26,9 +28,11 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/kubectl/pkg/util/term"
+	"k8s.io/kubernetes/pkg/client/conditions"
 	"k8s.io/utils/ptr"
 )
 
@@ -205,10 +209,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		Env:       env.Map(),
 		Namespace: namespace,
 	})
-	err = printResource(PrintInput{
+	err = writeResourceToFile(WriteResourceInput{
 		Object:       &secret,
 		Path:         filepath.Join(chartDir, "templates/secret.yaml"),
-		Name:         secret.Name,
 		Replacements: replacements,
 		AppendContents: `{{- with .Values.secrets }}
 data:
@@ -265,6 +268,7 @@ data:
 			PortMaps:         portMaps,
 			ProcessType:      processType,
 			Replicas:         replicaCountPlaceholder,
+			WorkingDir:       common.GetWorkingDir(appName, image),
 		})
 		if err != nil {
 			return fmt.Errorf("Error templating deployment: %w", err)
@@ -272,10 +276,9 @@ data:
 
 		replacements.Set(fmt.Sprintf("replicas: %d", replicaCountPlaceholder), fmt.Sprintf("replicas: {{.Values.processes.%s.replicas}}", processType))
 		deployments[processType] = deployment
-		err = printResource(PrintInput{
+		err = writeResourceToFile(WriteResourceInput{
 			Object:       &deployment,
 			Path:         filepath.Join(chartDir, fmt.Sprintf("templates/deployment-%s.yaml", deployment.Name)),
-			Name:         deployment.Name,
 			Replacements: replacements,
 		})
 		replacements.Delete(fmt.Sprintf("replicas: %d", replicaCountPlaceholder))
@@ -292,10 +295,9 @@ data:
 			PortMaps:  portMaps,
 		})
 
-		err := printResource(PrintInput{
+		err := writeResourceToFile(WriteResourceInput{
 			Object:       &service,
 			Path:         filepath.Join(chartDir, "templates/service-web.yaml"),
-			Name:         service.Name,
 			Replacements: replacements,
 		})
 		if err != nil {
@@ -473,27 +475,52 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		}
 	}
 
-	coreclient, err := corev1client.NewForConfig(&clientset.RestConfig)
+	entrypoint := ""
+	if selectedPod.Annotations["dokku.com/builder-type"] == "herokuish" {
+		entrypoint = "/exec"
+	}
+
+	return enterPod(ctx, EnterPodInput{
+		AppName:     appName,
+		Clientset:   clientset,
+		Command:     command,
+		Entrypoint:  entrypoint,
+		ProcessType: processType,
+		SelectedPod: selectedPod,
+	})
+}
+
+type EnterPodInput struct {
+	AppName     string
+	Clientset   KubernetesClient
+	Command     []string
+	Entrypoint  string
+	ProcessType string
+	SelectedPod v1.Pod
+}
+
+func enterPod(ctx context.Context, input EnterPodInput) error {
+	coreclient, err := corev1client.NewForConfig(&input.Clientset.RestConfig)
 	if err != nil {
 		return fmt.Errorf("Error creating corev1 client: %w", err)
 	}
 
 	req := coreclient.RESTClient().Post().
 		Resource("pods").
-		Namespace(selectedPod.Namespace).
-		Name(selectedPod.Name).
+		Namespace(input.SelectedPod.Namespace).
+		Name(input.SelectedPod.Name).
 		SubResource("exec")
 
-	req.Param("container", fmt.Sprintf("%s-%s", appName, processType))
+	req.Param("container", fmt.Sprintf("%s-%s", input.AppName, input.ProcessType))
 	req.Param("stdin", "true")
 	req.Param("stdout", "true")
 	req.Param("stderr", "true")
 	req.Param("tty", "true")
 
-	if selectedPod.Annotations["dokku.com/builder-type"] == "herokuish" {
-		req.Param("command", "/exec")
+	if input.Entrypoint != "" {
+		req.Param("command", input.Entrypoint)
 	}
-	for _, cmd := range command {
+	for _, cmd := range input.Command {
 		req.Param("command", cmd)
 	}
 
@@ -506,7 +533,7 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 	sizeQueue := t.MonitorSize(size)
 
 	return t.Safe(func() error {
-		exec, err := remotecommand.NewSPDYExecutor(&clientset.RestConfig, "POST", req.URL())
+		exec, err := remotecommand.NewSPDYExecutor(&input.Clientset.RestConfig, "POST", req.URL())
 		if err != nil {
 			return fmt.Errorf("Error creating executor: %w", err)
 		}
@@ -571,7 +598,7 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 	ch := make(chan bool)
 	podItems := podList.Items
 
-	if os.Getenv("FORCE_COLOR") == "1" {
+	if os.Getenv("FORCE_TTY") == "1" {
 		color.NoColor = false
 	}
 
@@ -692,3 +719,440 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 
 	return nil
 }
+
+// TriggerSchedulerRun runs a command in an ephemeral container
+func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+
+	extraEnv := map[string]string{}
+	if envCount > 0 {
+		var envPairs []string
+		envPairs, args = args[0:envCount], args[envCount:]
+		for _, envPair := range envPairs {
+			parts := strings.SplitN(envPair, "=", 2)
+			if len(parts) != 2 {
+				return fmt.Errorf("Invalid environment variable pair: %s", envPair)
+			}
+			extraEnv[parts[0]] = parts[1]
+		}
+	}
+
+	imageTag, err := common.GetRunningImageTag(appName, "")
+	if err != nil {
+		return fmt.Errorf("Error getting running image tag: %w", err)
+	}
+	imageRepo := common.GetAppImageRepo(appName)
+	image, err := common.GetDeployingAppImageName(appName, imageTag, imageRepo)
+	if err != nil {
+		return fmt.Errorf("Error getting deploying app image name: %w", err)
+	}
+
+	imageStage, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.image-stage\" }}")
+	if err != nil {
+		return fmt.Errorf("Error getting image stage: %w", err)
+	}
+	if imageStage != "release" {
+		common.LogWarn("Invalid image stage detected: expected 'release', got '$IMAGE_STAGE'")
+		return fmt.Errorf("Successfully deploy your app to fix dokku run calls")
+	}
+
+	dokkuRmContainer := os.Getenv("DOKKU_RM_CONTAINER")
+	if dokkuRmContainer == "" {
+		appRmContainer, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+			Trigger:       "config-get",
+			Args:          []string{appName, "DOKKU_RM_CONTAINER"},
+			CaptureOutput: true,
+			StreamStdio:   false,
+		})
+		if err != nil {
+			globalRmContainer, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+				Trigger:       "config-get-global",
+				Args:          []string{"DOKKU_RM_CONTAINER"},
+				CaptureOutput: true,
+				StreamStdio:   false,
+			})
+			if err == nil {
+				dokkuRmContainer = globalRmContainer.Stdout
+			}
+		} else {
+			dokkuRmContainer = appRmContainer.Stdout
+		}
+	}
+	if dokkuRmContainer == "" {
+		dokkuRmContainer = "true"
+	}
+
+	rmContainer, err := strconv.ParseBool(dokkuRmContainer)
+	if err != nil {
+		return fmt.Errorf("Error parsing DOKKU_RM_CONTAINER value as boolean: %w", err)
+	}
+
+	labels := map[string]string{
+		"dokku.com/app-name": appName,
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		extraEnv["TRACE"] = "true"
+	}
+
+	processType := "run"
+	if os.Getenv("DOKKU_CRON_ID") != "" {
+		processType = "cron"
+		labels["dokku.com/cron-id"] = os.Getenv("DOKKU_CRON_ID")
+	}
+
+	dockerArgs := []string{}
+	if b, err := common.PlugnTriggerSetup("docker-args-run", []string{appName, imageTag}...).SetInput("").Output(); err == nil {
+		words, err := shellquote.Split(strings.TrimSpace(string(b[:])))
+		if err != nil {
+			return err
+		}
+
+		dockerArgs = append(dockerArgs, words...)
+	}
+
+	imageSourceType, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.builder-type\" }}")
+	if err != nil {
+		return fmt.Errorf("Error getting image builder type: %w", err)
+	}
+
+	if b, err := common.PlugnTriggerSetup("docker-args-process-run", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err == nil {
+		words, err := shellquote.Split(strings.TrimSpace(string(b[:])))
+		if err != nil {
+			return err
+		}
+
+		dockerArgs = append(dockerArgs, words...)
+	}
+
+	// todo: do something with docker args
+	command := args
+	if len(args) == 0 {
+		command = []string{"/bin/bash"}
+		if globalShell, err := common.PlugnTriggerOutputAsString("config-get-global", []string{"DOKKU_APP_SHELL"}...); err == nil && globalShell != "" {
+			command = []string{globalShell}
+		}
+		if appShell, err := common.PlugnTriggerOutputAsString("config-get", []string{appName, "DOKKU_APP_SHELL"}...); err == nil && appShell != "" {
+			command = []string{appShell}
+		}
+	} else if len(args) == 1 {
+		resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+			Trigger:       "procfile-get-command",
+			Args:          []string{appName, args[0], "5000"},
+			CaptureOutput: true,
+			StreamStdio:   false,
+		})
+		if err == nil && resp.Stdout != "" {
+			common.LogInfo1Quiet(fmt.Sprintf("Found '%s' in Procfile, running that command", args[0]))
+			return err
+		}
+		// todo: run command in procfile
+	}
+
+	entrypoint := ""
+	if imageSourceType == "herokuish" {
+		entrypoint = "/exec"
+	}
+
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
+	if err != nil {
+		return fmt.Errorf("Error creating helm agent: %w", err)
+	}
+
+	values, err := helmAgent.GetValues(fmt.Sprintf("dokku-%s", appName))
+	if err != nil {
+		return fmt.Errorf("Error getting helm values: %w", err)
+	}
+
+	deploymentIDValue, ok := values["deploment_id"].(string)
+	if !ok {
+		return fmt.Errorf("Deployment ID is not a string")
+	}
+	if len(deploymentIDValue) == 0 {
+		return fmt.Errorf("Deployment ID is empty")
+	}
+	deploymentID, err := strconv.ParseInt(deploymentIDValue, 10, 64)
+	if err != nil {
+		return fmt.Errorf("Error parsing deployment ID: %w", err)
+	}
+
+	attachToPod := os.Getenv("DOKKU_DETACH_CONTAINER") != "1"
+	imagePullSecrets := common.PropertyGetDefault("scheduler-k3s", appName, "image-pull-secrets", "")
+	workingDir := common.GetWorkingDir(appName, image)
+	job, err := templateKubernetesJob(Job{
+		AppName:          appName,
+		Command:          command,
+		DeploymentID:     deploymentID,
+		Entrypoint:       entrypoint,
+		Env:              extraEnv,
+		Image:            image,
+		ImagePullSecrets: imagePullSecrets,
+		ImageSourceType:  imageSourceType,
+		Interactive:      attachToPod,
+		Labels:           labels,
+		Namespace:        namespace,
+		ProcessType:      processType,
+		RemoveContainer:  rmContainer,
+		WorkingDir:       workingDir,
+	})
+	if err != nil {
+		return fmt.Errorf("Error templating job: %w", err)
+	}
+
+	if os.Getenv("FORCE_TTY") == "1" {
+		color.NoColor = false
+	}
+
+	err = writeResourceToFile(WriteResourceInput{
+		Object: &job,
+		Path:   "/tmp/job.yaml",
+	})
+	if err != nil {
+		return fmt.Errorf("Error writing job: %w", err)
+	}
+	common.CatFile("/tmp/job.yaml")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
+	createdJob, err := createKubernetesJob(ctx, job)
+	if err != nil {
+		return fmt.Errorf("Error creating job: %w", err)
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
+	podList, err := WaitForPodToExist(ctx, WaitForPodToExistInput{
+		Clientset:  clientset,
+		Namespace:  namespace,
+		RetryCount: 3,
+		Selector:   batchJobSelector,
+	})
+	if err != nil {
+		return fmt.Errorf("Error waiting for pod to exist: %w", err)
+	}
+
+	for _, pod := range podList.Items {
+		common.LogQuiet(pod.Name)
+	}
+
+	if !attachToPod {
+		return nil
+	}
+
+	err = WaitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
+		Clientset: clientset,
+		Namespace: namespace,
+		Selector:  batchJobSelector,
+		Timeout:   300,
+		Waiter:    isPodReady,
+	})
+	if err != nil {
+		if errors.Is(err, conditions.ErrPodCompleted) {
+			podList, podErr := GetPod(ctx, GetPodInput{
+				Clientset: clientset,
+				Namespace: namespace,
+				Selector:  batchJobSelector,
+			})
+			if podErr != nil {
+				return fmt.Errorf("Error completed pod: %w", err)
+			}
+			selectedPod := podList.Items[0]
+			if selectedPod.Status.Phase == v1.PodFailed {
+				for _, status := range selectedPod.Status.ContainerStatuses {
+					if status.Name != fmt.Sprintf("%s-%s", appName, processType) {
+						continue
+					}
+					if status.State.Terminated == nil {
+						continue
+					}
+					return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code: %s", status.State.Terminated.Message)
+				}
+
+				return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code")
+			} else if selectedPod.Status.Phase == v1.PodSucceeded {
+				return errors.New("Unable to attach as the pod has already exited with a successful exit code")
+			}
+		}
+		return fmt.Errorf("Error waiting for pod to be running: %w", err)
+	}
+
+	podList, err = GetPod(ctx, GetPodInput{
+		Clientset: clientset,
+		Namespace: namespace,
+		Selector:  batchJobSelector,
+	})
+	if err != nil {
+		return fmt.Errorf("Error getting pod: %w", err)
+	}
+	selectedPod := podList.Items[0]
+
+	switch selectedPod.Status.Phase {
+	case v1.PodFailed, v1.PodSucceeded:
+		if selectedPod.Status.Phase == v1.PodFailed {
+			for _, status := range selectedPod.Status.ContainerStatuses {
+				if status.Name != fmt.Sprintf("%s-%s", appName, processType) {
+					continue
+				}
+				if status.State.Terminated == nil {
+					continue
+				}
+				return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code: %s", status.State.Terminated.Message)
+			}
+
+			return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code")
+		} else if selectedPod.Status.Phase == v1.PodSucceeded {
+			return errors.New("Unable to attach as the pod has already exited with a successful exit code")
+		}
+	case v1.PodRunning:
+		err := enterPod(ctx, EnterPodInput{
+			AppName:     appName,
+			Clientset:   clientset,
+			Command:     command,
+			Entrypoint:  entrypoint,
+			ProcessType: processType,
+			SelectedPod: selectedPod,
+		})
+		if err != nil {
+			return err
+		}
+
+		err = clientset.Client.BatchV1().Jobs(namespace).Delete(ctx, createdJob.Name, metav1.DeleteOptions{
+			PropagationPolicy: ptr.To(metav1.DeletePropagationForeground),
+		})
+		if err != nil {
+			return fmt.Errorf("Error deleting pod: %w", err)
+		}
+
+		return nil
+	default:
+		return fmt.Errorf("Unable to attach as the pod is in an unknown state: %s", selectedPod.Status.Phase)
+	}
+	// todo: support scheduler-post-run
+
+	return nil
+}
+
+func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
+	return func(ctx context.Context) (bool, error) {
+		fmt.Printf(".") // progress bar!
+
+		pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		switch pod.Status.Phase {
+		case v1.PodRunning:
+			return true, nil
+		case v1.PodFailed, v1.PodSucceeded:
+			return false, conditions.ErrPodCompleted
+		}
+		return false, nil
+	}
+}
+
+func isPodComplete(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
+	return func(ctx context.Context) (bool, error) {
+		fmt.Printf(".") // progress bar!
+
+		pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		switch pod.Status.Phase {
+		case v1.PodFailed, v1.PodSucceeded:
+			return true, nil
+		}
+		return false, nil
+	}
+}
+
+type WaitForPodBySelectorRunningInput struct {
+	Clientset KubernetesClient
+	Namespace string
+	Selector  string
+	Timeout   int
+	Waiter    func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
+}
+
+type WaitForPodToExistInput struct {
+	Clientset  KubernetesClient
+	Namespace  string
+	RetryCount int
+	Selector   string
+}
+
+func WaitForPodToExist(ctx context.Context, input WaitForPodToExistInput) (v1.PodList, error) {
+	var podList v1.PodList
+	var err error
+	for i := 0; i < input.RetryCount; i++ {
+		podList, err = GetPod(ctx, GetPodInput{
+			Clientset: input.Clientset,
+			Namespace: input.Namespace,
+			Selector:  input.Selector,
+		})
+		if err == nil {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	if err != nil {
+		return podList, fmt.Errorf("Error listing pods: %w", err)
+	}
+	return podList, nil
+}
+
+type GetPodInput struct {
+	Clientset KubernetesClient
+	Namespace string
+	Selector  string
+}
+
+func GetPod(ctx context.Context, input GetPodInput) (v1.PodList, error) {
+	listOptions := metav1.ListOptions{LabelSelector: input.Selector}
+	podList, err := input.Clientset.Client.CoreV1().Pods(input.Namespace).List(ctx, listOptions)
+	return *podList, err
+}
+
+func WaitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
+	podList, err := WaitForPodToExist(ctx, WaitForPodToExistInput{
+		Clientset:  input.Clientset,
+		Namespace:  input.Namespace,
+		RetryCount: 3,
+		Selector:   input.Selector,
+	})
+	if err != nil {
+		return fmt.Errorf("Error waiting for pod to exist: %w", err)
+	}
+
+	if len(podList.Items) == 0 {
+		return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.Selector)
+	}
+
+	timeout := time.Duration(input.Timeout) * time.Second
+	for _, pod := range podList.Items {
+		if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
+			print("\n")
+			return err
+		}
+	}
+	print("\n")
+	return nil
+}

From 3f7e1d161afb2dcbc00c138cd338e5128ad2ae88 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 05:34:35 -0500
Subject: [PATCH 025/113] chore: remove unused docker args code

---
 plugins/scheduler-k3s/triggers.go | 37 -------------------------------
 1 file changed, 37 deletions(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index cfff5cb92..c5eb25c04 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -19,7 +19,6 @@ import (
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/dokku/dokku/plugins/config"
 	"github.com/fatih/color"
-	"github.com/kballard/go-shellquote"
 	"github.com/rancher/wharfie/pkg/registries"
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
@@ -803,30 +802,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		labels["dokku.com/cron-id"] = os.Getenv("DOKKU_CRON_ID")
 	}
 
-	dockerArgs := []string{}
-	if b, err := common.PlugnTriggerSetup("docker-args-run", []string{appName, imageTag}...).SetInput("").Output(); err == nil {
-		words, err := shellquote.Split(strings.TrimSpace(string(b[:])))
-		if err != nil {
-			return err
-		}
-
-		dockerArgs = append(dockerArgs, words...)
-	}
-
 	imageSourceType, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.builder-type\" }}")
 	if err != nil {
 		return fmt.Errorf("Error getting image builder type: %w", err)
 	}
 
-	if b, err := common.PlugnTriggerSetup("docker-args-process-run", []string{appName, imageSourceType, imageTag}...).SetInput("").Output(); err == nil {
-		words, err := shellquote.Split(strings.TrimSpace(string(b[:])))
-		if err != nil {
-			return err
-		}
-
-		dockerArgs = append(dockerArgs, words...)
-	}
-
 	// todo: do something with docker args
 	command := args
 	if len(args) == 0 {
@@ -1067,23 +1047,6 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp
 	}
 }
 
-func isPodComplete(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
-	return func(ctx context.Context) (bool, error) {
-		fmt.Printf(".") // progress bar!
-
-		pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-
-		switch pod.Status.Phase {
-		case v1.PodFailed, v1.PodSucceeded:
-			return true, nil
-		}
-		return false, nil
-	}
-}
-
 type WaitForPodBySelectorRunningInput struct {
 	Clientset KubernetesClient
 	Namespace string

From 0127f836fa0932936e4809140846cc8c03aa7569 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 06:08:18 -0500
Subject: [PATCH 026/113] feat: implement cron jobs

---
 docs/deployment/schedulers/k3s.md |   2 +-
 plugins/scheduler-k3s/template.go | 156 +++++++++++++++++++++++++++++-
 plugins/scheduler-k3s/triggers.go |  44 ++++++++-
 3 files changed, 195 insertions(+), 7 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index ced87c825..815474024 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -162,6 +162,7 @@ The following sections describe implemented and unimplemented scheduler function
 This plugin implements various functionality through `plugn` triggers to integrate with Docker for running apps on a single server. The following functionality is supported by the `scheduler-docker-local` plugin.
 
 - `apps:destroy`
+- `cron`
 - `run`
        - The `scheduler-post-run` trigger is not always triggered
 - `run:detached`
@@ -180,7 +181,6 @@ This plugin implements various functionality through `plugn` triggers to integra
 
 The following Dokku functionality is not implemented at this time.
 
-- `cron`
 - `vector` log integration
 - persistent storage
 - healthchecks
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 53c582ac4..2c425692d 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -82,6 +82,7 @@ type Job struct {
 	Labels           map[string]string
 	Namespace        string
 	ProcessType      string
+	Schedule         string
 	RemoveContainer  bool
 	WorkingDir       string
 }
@@ -163,6 +164,154 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 	return nil
 }
 
+func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
+	if input.Schedule == "" {
+		return batchv1.CronJob{}, fmt.Errorf("Schedule cannot be empty")
+	}
+	labels := map[string]string{
+		"dokku.com/app-name":         input.AppName,
+		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"dokku.com/process-type":     input.ProcessType,
+	}
+	annotations := map[string]string{
+		"dokku.com/app-name":      input.AppName,
+		"dokku.com/builder-type":  input.ImageSourceType,
+		"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
+		"dokku.com/managed":       "true",
+		"dokku.com/process-type":  input.ProcessType,
+	}
+
+	for key, value := range input.Labels {
+		labels[key] = value
+	}
+	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
+
+	env := []corev1.EnvVar{}
+	for key, value := range input.Env {
+		env = append(env, corev1.EnvVar{
+			Name:  key,
+			Value: value,
+		})
+	}
+
+	job := batchv1.CronJob{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: fmt.Sprintf("%s-%s-", input.AppName, input.ProcessType),
+			Namespace:    input.Namespace,
+			Labels:       labels,
+			Annotations:  annotations,
+		},
+		Spec: batchv1.CronJobSpec{
+			ConcurrencyPolicy:          batchv1.AllowConcurrent,
+			FailedJobsHistoryLimit:     ptr.To(int32(10)),
+			Schedule:                   input.Schedule,
+			StartingDeadlineSeconds:    ptr.To(int64(60)),
+			SuccessfulJobsHistoryLimit: ptr.To(int32(10)),
+			Suspend:                    ptr.To(false),
+			TimeZone:                   ptr.To("Etc/UTC"),
+			JobTemplate: batchv1.JobTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      labels,
+					Annotations: annotations,
+				},
+				Spec: batchv1.JobSpec{
+					BackoffLimit:         ptr.To(int32(0)),
+					PodReplacementPolicy: ptr.To(batchv1.Failed),
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Args: input.Command,
+									Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+									Env:  env,
+									EnvFrom: []corev1.EnvFromSource{
+										{
+											SecretRef: &corev1.SecretEnvSource{
+												LocalObjectReference: corev1.LocalObjectReference{
+													Name: secretName,
+												},
+												Optional: ptr.To(true),
+											},
+										},
+									},
+									Image:           input.Image,
+									ImagePullPolicy: corev1.PullAlways,
+									Resources: corev1.ResourceRequirements{
+										Limits:   corev1.ResourceList{},
+										Requests: corev1.ResourceList{},
+									},
+									WorkingDir: input.WorkingDir,
+								},
+							},
+							RestartPolicy: corev1.RestartPolicyNever,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if input.Entrypoint != "" {
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command = []string{input.Entrypoint}
+	}
+
+	if input.RemoveContainer {
+		job.Spec.JobTemplate.Spec.TTLSecondsAfterFinished = ptr.To(int32(60))
+	}
+
+	if input.ImagePullSecrets != "" {
+		job.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
+			{
+				Name: input.ImagePullSecrets,
+			},
+		}
+	}
+
+	cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
+	if err != nil && cpuLimit != "" && cpuLimit != "0" {
+		cpuQuantity, err := resource.ParseQuantity(cpuLimit)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing cpu limit: %w", err)
+		}
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
+	}
+	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
+	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
+		nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
+		}
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
+	}
+	memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
+	if err != nil && memoryLimit != "" && memoryLimit != "0" {
+		memoryQuantity, err := resource.ParseQuantity(memoryLimit)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing memory limit: %w", err)
+		}
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
+	}
+
+	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
+	if err != nil && cpuRequest != "" && cpuRequest != "0" {
+		cpuQuantity, err := resource.ParseQuantity(cpuRequest)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing cpu request: %w", err)
+		}
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
+	}
+	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
+	if err != nil && memoryRequest != "" && memoryRequest != "0" {
+		memoryQuantity, err := resource.ParseQuantity(memoryRequest)
+		if err != nil {
+			return job, fmt.Errorf("Error parsing memory request: %w", err)
+		}
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
+	}
+
+	return job, nil
+}
+
 func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	labels := map[string]string{
 		"dokku.com/app-name":         input.AppName,
@@ -364,8 +513,6 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 		"dokku.com/process-type":  input.ProcessType,
 	}
 
-	// todo: implement a controller to delete pods on completion
-
 	for key, value := range input.Labels {
 		labels[key] = value
 	}
@@ -389,13 +536,16 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 		Spec: batchv1.JobSpec{
 			BackoffLimit: ptr.To(int32(0)),
 			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      labels,
+					Annotations: annotations,
+				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
 						{
 							Args: input.Command,
 							Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
 							Env:  env,
-							// todo: pull secret ref here
 							EnvFrom: []corev1.EnvFromSource{
 								{
 									SecretRef: &corev1.SecretEnvSource{
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index c5eb25c04..412109562 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -18,7 +18,9 @@ import (
 
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/dokku/dokku/plugins/config"
+	"github.com/dokku/dokku/plugins/cron"
 	"github.com/fatih/color"
+	"github.com/kballard/go-shellquote"
 	"github.com/rancher/wharfie/pkg/registries"
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
@@ -235,6 +237,7 @@ data:
 		}
 	}
 
+	workingDir := common.GetWorkingDir(appName, image)
 	deployments := map[string]appsv1.Deployment{}
 	i := 0
 	for processType := range processes {
@@ -267,7 +270,7 @@ data:
 			PortMaps:         portMaps,
 			ProcessType:      processType,
 			Replicas:         replicaCountPlaceholder,
-			WorkingDir:       common.GetWorkingDir(appName, image),
+			WorkingDir:       workingDir,
 		})
 		if err != nil {
 			return fmt.Errorf("Error templating deployment: %w", err)
@@ -280,11 +283,46 @@ data:
 			Path:         filepath.Join(chartDir, fmt.Sprintf("templates/deployment-%s.yaml", deployment.Name)),
 			Replacements: replacements,
 		})
-		replacements.Delete(fmt.Sprintf("replicas: %d", replicaCountPlaceholder))
-
 		if err != nil {
 			return fmt.Errorf("Error printing deployment: %w", err)
 		}
+
+		replacements.Delete(fmt.Sprintf("replicas: %d", replicaCountPlaceholder))
+	}
+
+	cronEntries, err := cron.FetchCronEntries(appName)
+	if err != nil {
+		return fmt.Errorf("Error fetching cron entries: %w", err)
+	}
+	for _, cronEntry := range cronEntries {
+		words, err := shellquote.Split(cronEntry.Command)
+		if err != nil {
+			return fmt.Errorf("Error parsing cron command: %w", err)
+		}
+		cronJob, err := templateKubernetesCronJob(Job{
+			AppName:          appName,
+			Command:          words,
+			Env:              map[string]string{},
+			Image:            image,
+			ImagePullSecrets: imagePullSecrets,
+			ImageSourceType:  imageSourceType,
+			Namespace:        namespace,
+			ProcessType:      cronEntry.ID,
+			Schedule:         cronEntry.Schedule,
+			WorkingDir:       workingDir,
+		})
+		if err != nil {
+			return fmt.Errorf("Error templating cron job: %w", err)
+		}
+
+		err = writeResourceToFile(WriteResourceInput{
+			Object:       &cronJob,
+			Path:         filepath.Join(chartDir, fmt.Sprintf("templates/cron-job-%s.yaml", cronEntry.ID)),
+			Replacements: replacements,
+		})
+		if err != nil {
+			return fmt.Errorf("Error printing cron job: %w", err)
+		}
 	}
 
 	if deployment, ok := deployments["web"]; ok {

From 277df036e9c54b2dd2cef098c0a9ed485cd0f73c Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 06:14:30 -0500
Subject: [PATCH 027/113] feat: implement default resource limits for cron
 tasks

---
 docs/deployment/schedulers/k3s.md | 6 ++++++
 plugins/scheduler-k3s/template.go | 8 ++++++++
 2 files changed, 14 insertions(+)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 815474024..fd80c27f3 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -195,10 +195,16 @@ The `docker-local` scheduler supports a minimal list of resource _limits_ and _r
 
 #### Resource Limits
 
+> [!NOTE]
+> Cron tasks retrieve resource limits based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM.
+
 - cpu: is specified in number of CPUs a process can access.
 - memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
 
 #### Resource Reservations
 
+> [!NOTE]
+> Cron tasks retrieve resource reservations based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM.
+
 - cpu: is specified in number of CPUs a process can access.
 - memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 2c425692d..eb1a13778 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -274,6 +274,8 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 			return job, fmt.Errorf("Error parsing cpu limit: %w", err)
 		}
 		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
+	} else {
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = resource.MustParse("500m")
 	}
 	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
 	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
@@ -290,6 +292,8 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 			return job, fmt.Errorf("Error parsing memory limit: %w", err)
 		}
 		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
+	} else {
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = resource.MustParse("512Mi")
 	}
 
 	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
@@ -299,6 +303,8 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 			return job, fmt.Errorf("Error parsing cpu request: %w", err)
 		}
 		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
+	} else {
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = resource.MustParse("500m")
 	}
 	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
 	if err != nil && memoryRequest != "" && memoryRequest != "0" {
@@ -307,6 +313,8 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 			return job, fmt.Errorf("Error parsing memory request: %w", err)
 		}
 		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
+	} else {
+		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = resource.MustParse("512Mi")
 	}
 
 	return job, nil

From bb8fd39e809818ce8273b84f76475575495b598d Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 06:14:44 -0500
Subject: [PATCH 028/113] chore: drop extra debugging for one-off jobs

---
 plugins/scheduler-k3s/triggers.go | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 412109562..24f8898e6 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -924,15 +924,6 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		color.NoColor = false
 	}
 
-	err = writeResourceToFile(WriteResourceInput{
-		Object: &job,
-		Path:   "/tmp/job.yaml",
-	})
-	if err != nil {
-		return fmt.Errorf("Error writing job: %w", err)
-	}
-	common.CatFile("/tmp/job.yaml")
-
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,

From 966ce76122e6b0e8548eca2a146e4cc405dfcea6 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 06:16:28 -0500
Subject: [PATCH 029/113] docs: add a note saying multi-cluster work isn't yet
 implemented

---
 docs/deployment/schedulers/k3s.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index fd80c27f3..7c29c26f6 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -91,6 +91,9 @@ At this point, all app deploys will be performed against the k3s cluster.
 
 ### Running a multi-cluster node
 
+> [!IMPORTANT]
+> This is future planned behavior. Commands here will not work as expected.
+
 For high-availability, it is recommended to add both agent and server nodes to the cluster. Dokku will default to starting the cluster with an embedded Etcd database backend, and is ready to add new agent or server nodes immediately.
 
 Multiple server nodes can be added with the `scheduler-k3s:add-server` command. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in server mode.

From 10e3bac24c4fce53b95b4e162c1f83c4e8a1df7e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 06:20:01 -0500
Subject: [PATCH 030/113] docs: remove references to docker-local

---
 docs/deployment/schedulers/k3s.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 7c29c26f6..c395a606a 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -162,7 +162,7 @@ The following sections describe implemented and unimplemented scheduler function
 
 ### Implemented Commands and Triggers
 
-This plugin implements various functionality through `plugn` triggers to integrate with Docker for running apps on a single server. The following functionality is supported by the `scheduler-docker-local` plugin.
+This plugin implements various functionality through `plugn` triggers to integrate with Docker for running apps on a single server. The following functionality is supported by the `scheduler-k3s` plugin.
 
 - `apps:destroy`
 - `cron`
@@ -194,7 +194,7 @@ App logs for the `logs` command are fetched by Dokku from running containers via
 
 ### Supported Resource Management Properties
 
-The `docker-local` scheduler supports a minimal list of resource _limits_ and _reservations_. The following properties are supported:
+The `k3s` scheduler supports a minimal list of resource _limits_ and _reservations_. The following properties are supported:
 
 #### Resource Limits
 

From 9d27ff39da5d5e0695ac150d9abb2a3bb8c24b49 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 06:38:12 -0500
Subject: [PATCH 031/113] refactor: only delete the helm chart in the
 scheduler-post-delete hook

---
 plugins/scheduler-k3s/Makefile                 |  2 +-
 plugins/scheduler-k3s/src/triggers/triggers.go |  4 ++++
 plugins/scheduler-k3s/triggers.go              | 12 ++++++++++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 3ea295207..776266dec 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set subcommands/show-kubeconfig
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-run triggers/scheduler-stop
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index 392dbb0e6..f0c64098a 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -89,6 +89,10 @@ func main() {
 		scheduler := flag.Arg(0)
 		appName := flag.Arg(1)
 		err = scheduler_k3s.TriggerSchedulerStop(scheduler, appName)
+	case "scheduler-post-delete":
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		err = scheduler_k3s.TriggerSchedulerPostDelete(scheduler, appName)
 	case "scheduler-run":
 		var envCount int
 		scheduler := flag.Arg(0)
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 24f8898e6..b709088b4 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -69,8 +69,11 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
 	return nil
 }
 
-// TriggerPostDelete destroys the scheduler-k3s data for a given app container
-func TriggerPostDelete(appName string) error {
+// TriggerSchedulerPostDelete destroys the scheduler-k3s data for a given app container
+func TriggerSchedulerPostDelete(scheduler string, appName string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
 	if err != nil {
@@ -82,6 +85,11 @@ func TriggerPostDelete(appName string) error {
 		return fmt.Errorf("Error uninstalling chart: %w", err)
 	}
 
+	return nil
+}
+
+// TriggerPostDelete destroys the scheduler-k3s data for a given app container
+func TriggerPostDelete(appName string) error {
 	dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName)
 	propertyErr := common.PropertyDestroy("scheduler-k3s", appName)
 

From 1c5505f78bb246015a26b899d79fd9b84daffdd3 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 15:11:51 -0500
Subject: [PATCH 032/113] refactor: move functions around and ensure we respect
 local versions of golang dependencies

---
 plugins/scheduler-k3s/functions.go | 181 +++++++++++++++++-
 plugins/scheduler-k3s/go.mod       |  12 +-
 plugins/scheduler-k3s/go.sum       |   6 +-
 plugins/scheduler-k3s/triggers.go  | 286 +++++++----------------------
 4 files changed, 255 insertions(+), 230 deletions(-)

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index d4fb376ac..7b95ddaa4 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -4,21 +4,43 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"time"
 
 	"github.com/dokku/dokku/plugins/common"
-	"github.com/dokku/dokku/plugins/config"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/kubectl/pkg/util/term"
+	"k8s.io/kubernetes/pkg/client/conditions"
 	"mvdan.cc/sh/v3/shell"
 )
 
+type EnterPodInput struct {
+	AppName     string
+	Clientset   KubernetesClient
+	Command     []string
+	Entrypoint  string
+	ProcessType string
+	SelectedPod v1.Pod
+}
+
+type GetPodInput struct {
+	Clientset KubernetesClient
+	Namespace string
+	Selector  string
+}
+
 // StartCommandInput contains all the information needed to get the start command
 type StartCommandInput struct {
 	// AppName is the name of the app
@@ -45,6 +67,21 @@ type KubernetesClient struct {
 	RestConfig rest.Config
 }
 
+type WaitForPodBySelectorRunningInput struct {
+	Clientset KubernetesClient
+	Namespace string
+	Selector  string
+	Timeout   int
+	Waiter    func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
+}
+
+type WaitForPodToExistInput struct {
+	Clientset  KubernetesClient
+	Namespace  string
+	RetryCount int
+	Selector   string
+}
+
 func NewKubernetesClient() (KubernetesClient, error) {
 	clientConfig := KubernetesClientConfig()
 	restConf, err := clientConfig.ClientConfig()
@@ -132,15 +169,69 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 	return nil
 }
 
+func enterPod(ctx context.Context, input EnterPodInput) error {
+	coreclient, err := corev1client.NewForConfig(&input.Clientset.RestConfig)
+	if err != nil {
+		return fmt.Errorf("Error creating corev1 client: %w", err)
+	}
+
+	req := coreclient.RESTClient().Post().
+		Resource("pods").
+		Namespace(input.SelectedPod.Namespace).
+		Name(input.SelectedPod.Name).
+		SubResource("exec")
+
+	req.Param("container", fmt.Sprintf("%s-%s", input.AppName, input.ProcessType))
+	req.Param("stdin", "true")
+	req.Param("stdout", "true")
+	req.Param("stderr", "true")
+	req.Param("tty", "true")
+
+	if input.Entrypoint != "" {
+		req.Param("command", input.Entrypoint)
+	}
+	for _, cmd := range input.Command {
+		req.Param("command", cmd)
+	}
+
+	t := term.TTY{
+		In:  os.Stdin,
+		Out: os.Stdout,
+		Raw: true,
+	}
+	size := t.GetSize()
+	sizeQueue := t.MonitorSize(size)
+
+	return t.Safe(func() error {
+		exec, err := remotecommand.NewSPDYExecutor(&input.Clientset.RestConfig, "POST", req.URL())
+		if err != nil {
+			return fmt.Errorf("Error creating executor: %w", err)
+		}
+
+		return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+			Stdin:             os.Stdin,
+			Stdout:            os.Stdout,
+			Stderr:            os.Stderr,
+			Tty:               true,
+			TerminalSizeQueue: sizeQueue,
+		})
+	})
+}
+
 func extractStartCommand(input StartCommandInput) string {
 	command := ""
 	if input.ImageSourceType == "herokuish" {
 		command = "/start " + input.ProcessType
 	}
 
-	startCommandOverride, ok := config.Get(input.AppName, "DOKKU_START_CMD")
-	if ok {
-		command = startCommandOverride
+	startCommandResp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Trigger:       "config-get",
+		Args:          []string{input.AppName, "DOKKU_START_CMD"},
+		CaptureOutput: true,
+		StreamStdio:   false,
+	})
+	if err == nil && startCommandResp.ExitCode == 0 {
+		command = startCommandResp.Stdout
 	}
 
 	if input.ImageSourceType == "herokuish" {
@@ -148,9 +239,14 @@ func extractStartCommand(input StartCommandInput) string {
 	}
 
 	if input.ImageSourceType == "dockerfile" {
-		startCommandOverride, ok := config.Get(input.AppName, "DOKKU_DOCKERFILE_START_CMD")
-		if ok {
-			command = startCommandOverride
+		startCommandDockerfileResp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+			Trigger:       "config-get",
+			Args:          []string{input.AppName, "DOKKU_DOCKERFILE_START_CMD"},
+			CaptureOutput: true,
+			StreamStdio:   false,
+		})
+		if err == nil && startCommandDockerfileResp.ExitCode == 0 {
+			command = startCommandDockerfileResp.Stdout
 		}
 	}
 
@@ -221,3 +317,74 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 		Command: fields,
 	}, nil
 }
+
+func getPod(ctx context.Context, input GetPodInput) (v1.PodList, error) {
+	listOptions := metav1.ListOptions{LabelSelector: input.Selector}
+	podList, err := input.Clientset.Client.CoreV1().Pods(input.Namespace).List(ctx, listOptions)
+	return *podList, err
+}
+
+func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
+	return func(ctx context.Context) (bool, error) {
+		fmt.Printf(".") // progress bar!
+
+		pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		switch pod.Status.Phase {
+		case v1.PodRunning:
+			return true, nil
+		case v1.PodFailed, v1.PodSucceeded:
+			return false, conditions.ErrPodCompleted
+		}
+		return false, nil
+	}
+}
+
+func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
+	podList, err := waitForPodToExist(ctx, WaitForPodToExistInput{
+		Clientset:  input.Clientset,
+		Namespace:  input.Namespace,
+		RetryCount: 3,
+		Selector:   input.Selector,
+	})
+	if err != nil {
+		return fmt.Errorf("Error waiting for pod to exist: %w", err)
+	}
+
+	if len(podList.Items) == 0 {
+		return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.Selector)
+	}
+
+	timeout := time.Duration(input.Timeout) * time.Second
+	for _, pod := range podList.Items {
+		if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
+			print("\n")
+			return err
+		}
+	}
+	print("\n")
+	return nil
+}
+
+func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) (v1.PodList, error) {
+	var podList v1.PodList
+	var err error
+	for i := 0; i < input.RetryCount; i++ {
+		podList, err = getPod(ctx, GetPodInput{
+			Clientset: input.Clientset,
+			Namespace: input.Namespace,
+			Selector:  input.Selector,
+		})
+		if err == nil {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	if err != nil {
+		return podList, fmt.Errorf("Error listing pods: %w", err)
+	}
+	return podList, nil
+}
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index e5912efa4..677660577 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -4,7 +4,8 @@ go 1.21
 
 require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
-	github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa
+	github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
+	github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
 	github.com/fatih/color v1.16.0
 	github.com/go-resty/resty/v2 v2.11.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
@@ -54,6 +55,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dokku/dokku/plugins/app-json v0.0.0-00010101000000-000000000000 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
@@ -111,6 +113,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -123,6 +126,7 @@ require (
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rubenv/sql-migrate v1.5.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryanuber/columnize v2.1.2+incompatible // indirect
@@ -169,4 +173,10 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
+replace github.com/dokku/dokku/plugins/app-json => ../app-json
+
 replace github.com/dokku/dokku/plugins/common => ../common
+
+replace github.com/dokku/dokku/plugins/config => ../config
+
+replace github.com/dokku/dokku/plugins/cron => ../cron
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index 79ff56cb4..e79fd1b0a 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -102,8 +102,6 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
-github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa h1:lIscxUTyCvJSHV9oY3JBQazshpvSc7H2kEBwIZddMfc=
-github.com/dokku/dokku/plugins/config v0.0.0-20240112043630-928c388f10fa/go.mod h1:7lDnO8LtGfKX1eFKAXunqGGTqwzJNaf6V7IkKhxlN2g=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -319,6 +317,8 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
+github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -373,6 +373,8 @@ github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S
 github.com/rancher/dynamiclistener v0.3.5/go.mod h1:dW/YF6/m2+uEyJ5VtEcd9THxda599HP6N9dSXk81+k0=
 github.com/rancher/wharfie v0.6.5 h1:Xxj/ki8d4BFeQBvyViW9jX8BqxaPlZwXvRD8BG2Mm/8=
 github.com/rancher/wharfie v0.6.5/go.mod h1:iGXn5Y9GEhsx1h/lPUXAklK4DoY4+6NOJF+Li1j248Y=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97 h1:3RPlVWzZ/PDqmVuf/FKHARG5EMid/tl7cv54Sw/QRVY=
 github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index b709088b4..6b6447b11 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -29,10 +29,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/tools/remotecommand"
-	"k8s.io/kubectl/pkg/util/term"
 	"k8s.io/kubernetes/pkg/client/conditions"
 	"k8s.io/utils/ptr"
 )
@@ -69,25 +65,6 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
 	return nil
 }
 
-// TriggerSchedulerPostDelete destroys the scheduler-k3s data for a given app container
-func TriggerSchedulerPostDelete(scheduler string, appName string) error {
-	if scheduler != "k3s" {
-		return nil
-	}
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
-	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
-	if err != nil {
-		return fmt.Errorf("Error creating helm agent: %w", err)
-	}
-
-	err = helmAgent.UninstallChart(fmt.Sprintf("dokku-%s", appName))
-	if err != nil {
-		return fmt.Errorf("Error uninstalling chart: %w", err)
-	}
-
-	return nil
-}
-
 // TriggerPostDelete destroys the scheduler-k3s data for a given app container
 func TriggerPostDelete(appName string) error {
 	dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName)
@@ -535,64 +512,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 	})
 }
 
-type EnterPodInput struct {
-	AppName     string
-	Clientset   KubernetesClient
-	Command     []string
-	Entrypoint  string
-	ProcessType string
-	SelectedPod v1.Pod
-}
-
-func enterPod(ctx context.Context, input EnterPodInput) error {
-	coreclient, err := corev1client.NewForConfig(&input.Clientset.RestConfig)
-	if err != nil {
-		return fmt.Errorf("Error creating corev1 client: %w", err)
-	}
-
-	req := coreclient.RESTClient().Post().
-		Resource("pods").
-		Namespace(input.SelectedPod.Namespace).
-		Name(input.SelectedPod.Name).
-		SubResource("exec")
-
-	req.Param("container", fmt.Sprintf("%s-%s", input.AppName, input.ProcessType))
-	req.Param("stdin", "true")
-	req.Param("stdout", "true")
-	req.Param("stderr", "true")
-	req.Param("tty", "true")
-
-	if input.Entrypoint != "" {
-		req.Param("command", input.Entrypoint)
-	}
-	for _, cmd := range input.Command {
-		req.Param("command", cmd)
-	}
-
-	t := term.TTY{
-		In:  os.Stdin,
-		Out: os.Stdout,
-		Raw: true,
-	}
-	size := t.GetSize()
-	sizeQueue := t.MonitorSize(size)
-
-	return t.Safe(func() error {
-		exec, err := remotecommand.NewSPDYExecutor(&input.Clientset.RestConfig, "POST", req.URL())
-		if err != nil {
-			return fmt.Errorf("Error creating executor: %w", err)
-		}
-
-		return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
-			Stdin:             os.Stdin,
-			Stdout:            os.Stdout,
-			Stderr:            os.Stderr,
-			Tty:               true,
-			TerminalSizeQueue: sizeQueue,
-		})
-	})
-}
-
 // TriggerSchedulerLogs displays logs for a given application
 func TriggerSchedulerLogs(scheduler string, appName string, processType string, tail bool, quiet bool, numLines int64) error {
 	if scheduler != "k3s" {
@@ -712,59 +631,6 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 	return nil
 }
 
-// TriggerSchedulerStop stops an application
-func TriggerSchedulerStop(scheduler string, appName string) error {
-	if scheduler != "k3s" {
-		return nil
-	}
-
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Error creating kubernetes client: %w", err)
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
-		syscall.SIGINT,
-		syscall.SIGQUIT,
-		syscall.SIGTERM)
-	go func() {
-		<-signals
-		cancel()
-	}()
-
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
-	deploymentList, err := clientset.Client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{
-		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
-	})
-	if err != nil {
-		return fmt.Errorf("Error listing deployments: %w", err)
-	}
-
-	for _, deployment := range deploymentList.Items {
-		processType, ok := deployment.Annotations["dokku.com/process-type"]
-		if !ok {
-			return fmt.Errorf("Deployment %s does not have a process type annotation", deployment.Name)
-		}
-		common.LogVerboseQuiet(fmt.Sprintf("Stopping %s process", processType))
-		_, err := clientset.Client.AppsV1().Deployments(namespace).UpdateScale(ctx, deployment.Name, &autoscalingv1.Scale{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      deployment.Name,
-				Namespace: namespace,
-			},
-			Spec: autoscalingv1.ScaleSpec{
-				Replicas: 0,
-			},
-		}, metav1.UpdateOptions{})
-		if err != nil {
-			return fmt.Errorf("Error updating deployment scale: %w", err)
-		}
-	}
-
-	return nil
-}
-
 // TriggerSchedulerRun runs a command in an ephemeral container
 func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []string) error {
 	if scheduler != "k3s" {
@@ -954,7 +820,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	}
 
 	batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
-	podList, err := WaitForPodToExist(ctx, WaitForPodToExistInput{
+	podList, err := waitForPodToExist(ctx, WaitForPodToExistInput{
 		Clientset:  clientset,
 		Namespace:  namespace,
 		RetryCount: 3,
@@ -972,7 +838,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		return nil
 	}
 
-	err = WaitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
+	err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
 		Clientset: clientset,
 		Namespace: namespace,
 		Selector:  batchJobSelector,
@@ -981,7 +847,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	})
 	if err != nil {
 		if errors.Is(err, conditions.ErrPodCompleted) {
-			podList, podErr := GetPod(ctx, GetPodInput{
+			podList, podErr := getPod(ctx, GetPodInput{
 				Clientset: clientset,
 				Namespace: namespace,
 				Selector:  batchJobSelector,
@@ -1009,7 +875,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		return fmt.Errorf("Error waiting for pod to be running: %w", err)
 	}
 
-	podList, err = GetPod(ctx, GetPodInput{
+	podList, err = getPod(ctx, GetPodInput{
 		Clientset: clientset,
 		Namespace: namespace,
 		Selector:  batchJobSelector,
@@ -1065,94 +931,74 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	return nil
 }
 
-func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
-	return func(ctx context.Context) (bool, error) {
-		fmt.Printf(".") // progress bar!
-
-		pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-
-		switch pod.Status.Phase {
-		case v1.PodRunning:
-			return true, nil
-		case v1.PodFailed, v1.PodSucceeded:
-			return false, conditions.ErrPodCompleted
-		}
-		return false, nil
-	}
-}
-
-type WaitForPodBySelectorRunningInput struct {
-	Clientset KubernetesClient
-	Namespace string
-	Selector  string
-	Timeout   int
-	Waiter    func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
-}
-
-type WaitForPodToExistInput struct {
-	Clientset  KubernetesClient
-	Namespace  string
-	RetryCount int
-	Selector   string
-}
-
-func WaitForPodToExist(ctx context.Context, input WaitForPodToExistInput) (v1.PodList, error) {
-	var podList v1.PodList
-	var err error
-	for i := 0; i < input.RetryCount; i++ {
-		podList, err = GetPod(ctx, GetPodInput{
-			Clientset: input.Clientset,
-			Namespace: input.Namespace,
-			Selector:  input.Selector,
-		})
-		if err == nil {
-			break
-		}
-		time.Sleep(1 * time.Second)
+// TriggerSchedulerPostDelete destroys the scheduler-k3s data for a given app container
+func TriggerSchedulerPostDelete(scheduler string, appName string) error {
+	if scheduler != "k3s" {
+		return nil
 	}
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
 	if err != nil {
-		return podList, fmt.Errorf("Error listing pods: %w", err)
+		return fmt.Errorf("Error creating helm agent: %w", err)
 	}
-	return podList, nil
-}
 
-type GetPodInput struct {
-	Clientset KubernetesClient
-	Namespace string
-	Selector  string
-}
-
-func GetPod(ctx context.Context, input GetPodInput) (v1.PodList, error) {
-	listOptions := metav1.ListOptions{LabelSelector: input.Selector}
-	podList, err := input.Clientset.Client.CoreV1().Pods(input.Namespace).List(ctx, listOptions)
-	return *podList, err
-}
-
-func WaitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
-	podList, err := WaitForPodToExist(ctx, WaitForPodToExistInput{
-		Clientset:  input.Clientset,
-		Namespace:  input.Namespace,
-		RetryCount: 3,
-		Selector:   input.Selector,
-	})
+	err = helmAgent.UninstallChart(fmt.Sprintf("dokku-%s", appName))
 	if err != nil {
-		return fmt.Errorf("Error waiting for pod to exist: %w", err)
+		return fmt.Errorf("Error uninstalling chart: %w", err)
+	}
+
+	return nil
+}
+
+// TriggerSchedulerStop stops an application
+func TriggerSchedulerStop(scheduler string, appName string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	deploymentList, err := clientset.Client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing deployments: %w", err)
+	}
+
+	for _, deployment := range deploymentList.Items {
+		processType, ok := deployment.Annotations["dokku.com/process-type"]
+		if !ok {
+			return fmt.Errorf("Deployment %s does not have a process type annotation", deployment.Name)
+		}
+		common.LogVerboseQuiet(fmt.Sprintf("Stopping %s process", processType))
+		_, err := clientset.Client.AppsV1().Deployments(namespace).UpdateScale(ctx, deployment.Name, &autoscalingv1.Scale{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      deployment.Name,
+				Namespace: namespace,
+			},
+			Spec: autoscalingv1.ScaleSpec{
+				Replicas: 0,
+			},
+		}, metav1.UpdateOptions{})
+		if err != nil {
+			return fmt.Errorf("Error updating deployment scale: %w", err)
+		}
 	}
 
-	if len(podList.Items) == 0 {
-		return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.Selector)
-	}
-
-	timeout := time.Duration(input.Timeout) * time.Second
-	for _, pod := range podList.Items {
-		if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
-			print("\n")
-			return err
-		}
-	}
-	print("\n")
 	return nil
 }

From 297853781ff902c926ec103a7c6ad462b553ffa7 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 16:14:33 -0500
Subject: [PATCH 033/113] refactor: move all k8s code behind client

Also fix an issue where attaching to pods fails in rare cases
---
 plugins/scheduler-k3s/functions.go | 112 +++----------
 plugins/scheduler-k3s/helm.go      |   6 +-
 plugins/scheduler-k3s/k8s.go       | 257 +++++++++++++++++++++++++++++
 plugins/scheduler-k3s/template.go  |  16 +-
 plugins/scheduler-k3s/triggers.go  | 112 ++++++-------
 5 files changed, 345 insertions(+), 158 deletions(-)
 create mode 100644 plugins/scheduler-k3s/k8s.go

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 7b95ddaa4..bec2feb88 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -8,18 +8,11 @@ import (
 	"time"
 
 	"github.com/dokku/dokku/plugins/common"
-	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/kubectl/pkg/util/term"
 	"k8s.io/kubernetes/pkg/client/conditions"
@@ -35,12 +28,6 @@ type EnterPodInput struct {
 	SelectedPod v1.Pod
 }
 
-type GetPodInput struct {
-	Clientset KubernetesClient
-	Namespace string
-	Selector  string
-}
-
 // StartCommandInput contains all the information needed to get the start command
 type StartCommandInput struct {
 	// AppName is the name of the app
@@ -61,12 +48,6 @@ type StartCommandOutput struct {
 	Command []string
 }
 
-type KubernetesClient struct {
-	Client     kubernetes.Clientset
-	RestClient rest.Interface
-	RestConfig rest.Config
-}
-
 type WaitForPodBySelectorRunningInput struct {
 	Clientset KubernetesClient
 	Namespace string
@@ -82,69 +63,18 @@ type WaitForPodToExistInput struct {
 	Selector   string
 }
 
-func NewKubernetesClient() (KubernetesClient, error) {
-	clientConfig := KubernetesClientConfig()
-	restConf, err := clientConfig.ClientConfig()
-	if err != nil {
-		return KubernetesClient{}, err
-	}
-
-	restConf.GroupVersion = &schema.GroupVersion{
-		Group:   "api",
-		Version: "v1",
-	}
-
-	client, err := kubernetes.NewForConfig(restConf)
-	if err != nil {
-		return KubernetesClient{}, err
-	}
-
-	restConf.NegotiatedSerializer = runtime.NewSimpleNegotiatedSerializer(runtime.SerializerInfo{})
-
-	restClient, err := rest.RESTClientFor(restConf)
-	if err != nil {
-		return KubernetesClient{}, err
-	}
-
-	return KubernetesClient{
-		Client:     *client,
-		RestConfig: *restConf,
-		RestClient: restClient,
-	}, nil
-}
-
-func KubernetesClientConfig() clientcmd.ClientConfig {
-	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		&clientcmd.ClientConfigLoadingRules{ExplicitPath: KubeConfigPath},
-		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
-}
-
-func createKubernetesJob(ctx context.Context, job batchv1.Job) (batchv1.Job, error) {
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return batchv1.Job{}, err
-	}
-
-	createdJob, err := clientset.Client.BatchV1().Jobs(job.Namespace).Create(ctx, &job, metav1.CreateOptions{})
-	if err != nil {
-		return batchv1.Job{}, err
-	}
-
-	return *createdJob, nil
-}
-
 func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return err
 	}
 
-	namespaces, err := clientset.Client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	namespaces, err := clientset.ListNamespaces(ctx)
 	if err != nil {
 		return err
 	}
 
-	for _, namespace := range namespaces.Items {
+	for _, namespace := range namespaces {
 		if namespace.Name == namespaceName {
 			return nil
 		}
@@ -161,7 +91,9 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 			},
 		},
 	}
-	_, err = clientset.Client.CoreV1().Namespaces().Create(ctx, &namespace, metav1.CreateOptions{})
+	_, err = clientset.CreateNamespace(ctx, CreateNamespaceInput{
+		Namespace: namespace,
+	})
 	if err != nil {
 		return err
 	}
@@ -318,17 +250,14 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
-func getPod(ctx context.Context, input GetPodInput) (v1.PodList, error) {
-	listOptions := metav1.ListOptions{LabelSelector: input.Selector}
-	podList, err := input.Clientset.Client.CoreV1().Pods(input.Namespace).List(ctx, listOptions)
-	return *podList, err
-}
-
 func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
 	return func(ctx context.Context) (bool, error) {
 		fmt.Printf(".") // progress bar!
 
-		pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
+		pod, err := clientset.GetPod(ctx, GetPodInput{
+			Name:      podName,
+			Namespace: namespace,
+		})
 		if err != nil {
 			return false, err
 		}
@@ -344,7 +273,7 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp
 }
 
 func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
-	podList, err := waitForPodToExist(ctx, WaitForPodToExistInput{
+	pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
 		Clientset:  input.Clientset,
 		Namespace:  input.Namespace,
 		RetryCount: 3,
@@ -354,29 +283,28 @@ func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelector
 		return fmt.Errorf("Error waiting for pod to exist: %w", err)
 	}
 
-	if len(podList.Items) == 0 {
+	if len(pods) == 0 {
 		return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.Selector)
 	}
 
 	timeout := time.Duration(input.Timeout) * time.Second
-	for _, pod := range podList.Items {
+	for _, pod := range pods {
 		if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
 			print("\n")
-			return err
+			return fmt.Errorf("Error waiting for pod to be ready: %w", err)
 		}
 	}
 	print("\n")
 	return nil
 }
 
-func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) (v1.PodList, error) {
-	var podList v1.PodList
+func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) ([]v1.Pod, error) {
+	var pods []v1.Pod
 	var err error
 	for i := 0; i < input.RetryCount; i++ {
-		podList, err = getPod(ctx, GetPodInput{
-			Clientset: input.Clientset,
-			Namespace: input.Namespace,
-			Selector:  input.Selector,
+		pods, err = input.Clientset.ListPods(ctx, ListPodsInput{
+			Namespace:     input.Namespace,
+			LabelSelector: input.Selector,
 		})
 		if err == nil {
 			break
@@ -384,7 +312,7 @@ func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) (v1.Po
 		time.Sleep(1 * time.Second)
 	}
 	if err != nil {
-		return podList, fmt.Errorf("Error listing pods: %w", err)
+		return pods, fmt.Errorf("Error listing pods: %w", err)
 	}
-	return podList, nil
+	return pods, nil
 }
diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go
index b846bf154..067bbffd6 100644
--- a/plugins/scheduler-k3s/helm.go
+++ b/plugins/scheduler-k3s/helm.go
@@ -17,7 +17,6 @@ import (
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/kube"
 	"helm.sh/helm/v3/pkg/storage/driver"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var DevNullPrinter = func(format string, v ...interface{}) {}
@@ -106,7 +105,10 @@ func (h *HelmAgent) DeleteRevision(releaseName string, revision int) error {
 	}
 
 	secretName := fmt.Sprintf("sh.helm.release.v1.%s.v%d", releaseName, revision)
-	err = clientset.Client.CoreV1().Secrets(h.Namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{})
+	err = clientset.DeleteSecret(context.Background(), DeleteSecretInput{
+		Name:      secretName,
+		Namespace: h.Namespace,
+	})
 	if err != nil {
 		return fmt.Errorf("Error deleting secret: %w", err)
 	}
diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go
new file mode 100644
index 000000000..6c35d2cb4
--- /dev/null
+++ b/plugins/scheduler-k3s/k8s.go
@@ -0,0 +1,257 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"errors"
+
+	appsv1 "k8s.io/api/apps/v1"
+	autoscalingv1 "k8s.io/api/autoscaling/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/utils/ptr"
+)
+
+// KubernetesClient is a wrapper around the Kubernetes client
+type KubernetesClient struct {
+	// Client is the Kubernetes client
+	Client kubernetes.Clientset
+
+	// RestClient is the Kubernetes REST client
+	RestClient rest.Interface
+
+	// RestConfig is the Kubernetes REST config
+	RestConfig rest.Config
+}
+
+// NewKubernetesClient creates a new Kubernetes client
+func NewKubernetesClient() (KubernetesClient, error) {
+	clientConfig := KubernetesClientConfig()
+	restConf, err := clientConfig.ClientConfig()
+	if err != nil {
+		return KubernetesClient{}, err
+	}
+
+	restConf.GroupVersion = &schema.GroupVersion{
+		Group:   "api",
+		Version: "v1",
+	}
+
+	client, err := kubernetes.NewForConfig(restConf)
+	if err != nil {
+		return KubernetesClient{}, err
+	}
+
+	restConf.NegotiatedSerializer = runtime.NewSimpleNegotiatedSerializer(runtime.SerializerInfo{})
+
+	restClient, err := rest.RESTClientFor(restConf)
+	if err != nil {
+		return KubernetesClient{}, err
+	}
+
+	return KubernetesClient{
+		Client:     *client,
+		RestConfig: *restConf,
+		RestClient: restClient,
+	}, nil
+}
+
+// KubernetesClientConfig returns a Kubernetes client config
+func KubernetesClientConfig() clientcmd.ClientConfig {
+	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: KubeConfigPath},
+		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
+}
+
+// CreateJobInput contains all the information needed to create a Kubernetes job
+type CreateJobInput struct {
+	// Job is the Kubernetes job
+	Job batchv1.Job
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// CreateJob creates a Kubernetes job
+func (k KubernetesClient) CreateJob(ctx context.Context, input CreateJobInput) (batchv1.Job, error) {
+	job, err := k.Client.BatchV1().Jobs(input.Namespace).Create(ctx, &input.Job, metav1.CreateOptions{})
+	if err != nil {
+		return batchv1.Job{}, err
+	}
+
+	if job == nil {
+		return batchv1.Job{}, errors.New("job is nil")
+	}
+
+	return *job, err
+}
+
+// CreateNamespaceInput contains all the information needed to create a Kubernetes namespace
+type CreateNamespaceInput struct {
+	// Namespace is the Kubernetes namespace
+	Namespace v1.Namespace
+}
+
+// CreateNamespace creates a Kubernetes namespace
+func (k KubernetesClient) CreateNamespace(ctx context.Context, input CreateNamespaceInput) (v1.Namespace, error) {
+	namespace, err := k.Client.CoreV1().Namespaces().Create(ctx, &input.Namespace, metav1.CreateOptions{})
+	if err != nil {
+		return v1.Namespace{}, err
+	}
+
+	if namespace == nil {
+		return v1.Namespace{}, errors.New("namespace is nil")
+	}
+
+	return *namespace, err
+}
+
+// DeleteJobInput contains all the information needed to delete a Kubernetes job
+type DeleteJobInput struct {
+	// Name is the Kubernetes job name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// DeleteJob deletes a Kubernetes job
+func (k KubernetesClient) DeleteJob(ctx context.Context, input DeleteJobInput) error {
+	return k.Client.BatchV1().Jobs(input.Namespace).Delete(ctx, input.Name, metav1.DeleteOptions{
+		PropagationPolicy: ptr.To(metav1.DeletePropagationForeground),
+	})
+}
+
+// DeleteSecretInput contains all the information needed to delete a Kubernetes secret
+type DeleteSecretInput struct {
+	// Name is the Kubernetes secret name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// DeleteSecret deletes a Kubernetes secret
+func (k KubernetesClient) DeleteSecret(ctx context.Context, input DeleteSecretInput) error {
+	return k.Client.CoreV1().Secrets(input.Namespace).Delete(ctx, input.Name, metav1.DeleteOptions{})
+}
+
+// GetJobInput contains all the information needed to get a Kubernetes job
+type GetPodInput struct {
+	// Name is the Kubernetes pod name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// GetJob gets a Kubernetes job
+func (k KubernetesClient) GetPod(ctx context.Context, input GetPodInput) (v1.Pod, error) {
+	pod, err := k.Client.CoreV1().Pods(input.Namespace).Get(ctx, input.Name, metav1.GetOptions{})
+	if err != nil {
+		return v1.Pod{}, err
+	}
+
+	if pod == nil {
+		return v1.Pod{}, errors.New("pod is nil")
+	}
+
+	return *pod, err
+}
+
+// ListDeploymentsInput contains all the information needed to list Kubernetes deployments
+type ListDeploymentsInput struct {
+	// Namespace is the Kubernetes namespace
+	Namespace string
+
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+}
+
+// ListDeployments lists Kubernetes deployments
+func (k KubernetesClient) ListDeployments(ctx context.Context, input ListDeploymentsInput) ([]appsv1.Deployment, error) {
+	listOptions := metav1.ListOptions{LabelSelector: input.LabelSelector}
+	deployments, err := k.Client.AppsV1().Deployments(input.Namespace).List(ctx, listOptions)
+	if err != nil {
+		return []appsv1.Deployment{}, err
+	}
+
+	if deployments == nil {
+		return []appsv1.Deployment{}, errors.New("deployments is nil")
+	}
+
+	return deployments.Items, nil
+}
+
+// ListNamespaces lists Kubernetes namespaces
+func (k KubernetesClient) ListNamespaces(ctx context.Context) ([]v1.Namespace, error) {
+	namespaces, err := k.Client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return []v1.Namespace{}, err
+	}
+	if namespaces == nil {
+		return []v1.Namespace{}, errors.New("namespaces is nil")
+	}
+
+	return namespaces.Items, nil
+}
+
+// ListPodsInput contains all the information needed to list Kubernetes pods
+type ListPodsInput struct {
+	// Namespace is the Kubernetes namespace
+	Namespace string
+
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+}
+
+// ListPods lists Kubernetes pods
+func (k KubernetesClient) ListPods(ctx context.Context, input ListPodsInput) ([]v1.Pod, error) {
+	listOptions := metav1.ListOptions{LabelSelector: input.LabelSelector}
+	podList, err := k.Client.CoreV1().Pods(input.Namespace).List(ctx, listOptions)
+	if err != nil {
+		return []v1.Pod{}, err
+	}
+
+	if podList == nil {
+		return []v1.Pod{}, errors.New("pod list is nil")
+	}
+
+	return podList.Items, err
+}
+
+// ScaleDeploymentInput contains all the information needed to scale a Kubernetes deployment
+type ScaleDeploymentInput struct {
+	// Name is the Kubernetes deployment name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+
+	// Replicas is the number of replicas to scale to
+	Replicas int32
+}
+
+// ScaleDeployment scales a Kubernetes deployment
+func (k KubernetesClient) ScaleDeployment(ctx context.Context, input ScaleDeploymentInput) error {
+	_, err := k.Client.AppsV1().Deployments(input.Namespace).UpdateScale(ctx, input.Name, &autoscalingv1.Scale{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      input.Name,
+			Namespace: input.Namespace,
+		},
+		Spec: autoscalingv1.ScaleSpec{
+			Replicas: input.Replicas,
+		},
+	}, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index eb1a13778..1da582acc 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -1,6 +1,7 @@
 package scheduler_k3s
 
 import (
+	"crypto/rand"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -534,12 +535,19 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 		})
 	}
 
+	n := 5
+	b := make([]byte, n)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	suffix := strings.ToLower(fmt.Sprintf("%X", b))
+
 	job := batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: fmt.Sprintf("%s-%s-", input.AppName, input.ProcessType),
-			Namespace:    input.Namespace,
-			Labels:       labels,
-			Annotations:  annotations,
+			Name:        fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
+			Namespace:   input.Namespace,
+			Labels:      labels,
+			Annotations: annotations,
 		},
 		Spec: batchv1.JobSpec{
 			BackoffLimit: ptr.To(int32(0)),
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 6b6447b11..f3f34713b 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -10,7 +10,6 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -25,10 +24,8 @@ import (
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
-	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/client/conditions"
 	"k8s.io/utils/ptr"
 )
@@ -444,41 +441,33 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		labelSelector = append(labelSelector, fmt.Sprintf("dokku.com/process-type=%s", processType))
 	}
 
-	podList, err := clientset.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+	pods, err := clientset.ListPods(ctx, ListPodsInput{
+		Namespace:     namespace,
 		LabelSelector: strings.Join(labelSelector, ","),
 	})
 	if err != nil {
 		return fmt.Errorf("Error listing pods: %w", err)
 	}
 
-	if podList.Items == nil {
+	if len(pods) == 0 {
 		return fmt.Errorf("No pods found for app %s", appName)
 	}
-	if len(podList.Items) == 0 {
-		return fmt.Errorf("No pods found for app %s", appName)
-	}
-
-	pods := []string{}
-	for _, pod := range podList.Items {
-		pods = append(pods, pod.Name)
-	}
-	slices.Sort(pods)
 
 	processIndex--
-	if processIndex > len(podList.Items) {
+	if processIndex > len(pods) {
 		return fmt.Errorf("Process index %d out of range for app %s", processIndex, appName)
 	}
 
 	var selectedPod corev1.Pod
 	if podName != "" {
-		for _, pod := range podList.Items {
+		for _, pod := range pods {
 			if pod.Name == podName {
 				selectedPod = pod
 				break
 			}
 		}
 	} else {
-		selectedPod = podList.Items[processIndex]
+		selectedPod = pods[processIndex]
 	}
 
 	processType, ok := selectedPod.Labels["dokku.com/process-type"]
@@ -549,18 +538,18 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 	}
 
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
-	podList, err := clientset.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+	pods, err := clientset.ListPods(ctx, ListPodsInput{
+		Namespace:     namespace,
 		LabelSelector: strings.Join(labelSelector, ","),
 	})
 	if err != nil {
 		return fmt.Errorf("Error listing pods: %w", err)
 	}
-	if len(podList.Items) == 0 {
+	if len(pods) == 0 {
 		return fmt.Errorf("No pods found for app %s", appName)
 	}
 
 	ch := make(chan bool)
-	podItems := podList.Items
 
 	if os.Getenv("FORCE_TTY") == "1" {
 		color.NoColor = false
@@ -575,7 +564,7 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		color.FgMagenta,
 	}
 	// colorIndex := 0
-	for i := 0; i < len(podItems); i++ {
+	for i := 0; i < len(pods); i++ {
 		if processIndex > 0 && i != (processIndex-1) {
 			continue
 		}
@@ -589,7 +578,7 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 
 		podColor := colors[i%len(colors)]
 		dynoText := color.New(podColor).SprintFunc()
-		podName := podItems[i].Name
+		podName := pods[i].Name
 		podLogs, err := clientset.Client.CoreV1().Pods(namespace).GetLogs(podName, &logOptions).Stream(ctx)
 		if err != nil {
 			return err
@@ -798,6 +787,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		color.NoColor = false
 	}
 
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -806,21 +800,34 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		syscall.SIGTERM)
 	go func() {
 		<-signals
+		if attachToPod {
+			clientset.DeleteJob(ctx, DeleteJobInput{ // nolint: errcheck
+				Name:      job.Name,
+				Namespace: namespace,
+			})
+		}
 		cancel()
 	}()
 
-	createdJob, err := createKubernetesJob(ctx, job)
+	createdJob, err := clientset.CreateJob(ctx, CreateJobInput{
+		Job:       job,
+		Namespace: namespace,
+	})
 	if err != nil {
 		return fmt.Errorf("Error creating job: %w", err)
 	}
 
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	if attachToPod {
+		defer func() {
+			clientset.DeleteJob(ctx, DeleteJobInput{ // nolint: errcheck
+				Name:      job.Name,
+				Namespace: namespace,
+			})
+		}()
 	}
 
 	batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
-	podList, err := waitForPodToExist(ctx, WaitForPodToExistInput{
+	pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
 		Clientset:  clientset,
 		Namespace:  namespace,
 		RetryCount: 3,
@@ -830,7 +837,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		return fmt.Errorf("Error waiting for pod to exist: %w", err)
 	}
 
-	for _, pod := range podList.Items {
+	for _, pod := range pods {
 		common.LogQuiet(pod.Name)
 	}
 
@@ -847,15 +854,14 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	})
 	if err != nil {
 		if errors.Is(err, conditions.ErrPodCompleted) {
-			podList, podErr := getPod(ctx, GetPodInput{
-				Clientset: clientset,
-				Namespace: namespace,
-				Selector:  batchJobSelector,
+			pods, podErr := clientset.ListPods(ctx, ListPodsInput{
+				Namespace:     namespace,
+				LabelSelector: batchJobSelector,
 			})
 			if podErr != nil {
 				return fmt.Errorf("Error completed pod: %w", err)
 			}
-			selectedPod := podList.Items[0]
+			selectedPod := pods[0]
 			if selectedPod.Status.Phase == v1.PodFailed {
 				for _, status := range selectedPod.Status.ContainerStatuses {
 					if status.Name != fmt.Sprintf("%s-%s", appName, processType) {
@@ -875,15 +881,14 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		return fmt.Errorf("Error waiting for pod to be running: %w", err)
 	}
 
-	podList, err = getPod(ctx, GetPodInput{
-		Clientset: clientset,
-		Namespace: namespace,
-		Selector:  batchJobSelector,
+	pods, err = clientset.ListPods(ctx, ListPodsInput{
+		Namespace:     namespace,
+		LabelSelector: batchJobSelector,
 	})
 	if err != nil {
 		return fmt.Errorf("Error getting pod: %w", err)
 	}
-	selectedPod := podList.Items[0]
+	selectedPod := pods[0]
 
 	switch selectedPod.Status.Phase {
 	case v1.PodFailed, v1.PodSucceeded:
@@ -903,7 +908,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 			return errors.New("Unable to attach as the pod has already exited with a successful exit code")
 		}
 	case v1.PodRunning:
-		err := enterPod(ctx, EnterPodInput{
+		return enterPod(ctx, EnterPodInput{
 			AppName:     appName,
 			Clientset:   clientset,
 			Command:     command,
@@ -911,16 +916,6 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 			ProcessType: processType,
 			SelectedPod: selectedPod,
 		})
-		if err != nil {
-			return err
-		}
-
-		err = clientset.Client.BatchV1().Jobs(namespace).Delete(ctx, createdJob.Name, metav1.DeleteOptions{
-			PropagationPolicy: ptr.To(metav1.DeletePropagationForeground),
-		})
-		if err != nil {
-			return fmt.Errorf("Error deleting pod: %w", err)
-		}
 
 		return nil
 	default:
@@ -973,28 +968,25 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 	}()
 
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
-	deploymentList, err := clientset.Client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{
+	deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
+		Namespace:     namespace,
 		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
 	})
 	if err != nil {
 		return fmt.Errorf("Error listing deployments: %w", err)
 	}
 
-	for _, deployment := range deploymentList.Items {
+	for _, deployment := range deployments {
 		processType, ok := deployment.Annotations["dokku.com/process-type"]
 		if !ok {
 			return fmt.Errorf("Deployment %s does not have a process type annotation", deployment.Name)
 		}
 		common.LogVerboseQuiet(fmt.Sprintf("Stopping %s process", processType))
-		_, err := clientset.Client.AppsV1().Deployments(namespace).UpdateScale(ctx, deployment.Name, &autoscalingv1.Scale{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      deployment.Name,
-				Namespace: namespace,
-			},
-			Spec: autoscalingv1.ScaleSpec{
-				Replicas: 0,
-			},
-		}, metav1.UpdateOptions{})
+		err := clientset.ScaleDeployment(ctx, ScaleDeploymentInput{
+			Name:      deployment.Name,
+			Namespace: namespace,
+			Replicas:  0,
+		})
 		if err != nil {
 			return fmt.Errorf("Error updating deployment scale: %w", err)
 		}

From e4855941feed3156245081b8147f2cf4b054ac9f Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 16:51:33 -0500
Subject: [PATCH 034/113] feat: allow for stable cron names

The cron id is too long to store in the cronjob, so we generate a short identifier and look that up each time on deploy to ensure stability across time.
---
 plugins/scheduler-k3s/k8s.go      | 28 +++++++++++++++++++++++
 plugins/scheduler-k3s/template.go | 37 +++++++++++++++++++++++--------
 plugins/scheduler-k3s/triggers.go | 31 +++++++++++++++++++++++---
 3 files changed, 84 insertions(+), 12 deletions(-)

diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go
index 6c35d2cb4..c3b5766e9 100644
--- a/plugins/scheduler-k3s/k8s.go
+++ b/plugins/scheduler-k3s/k8s.go
@@ -165,6 +165,34 @@ func (k KubernetesClient) GetPod(ctx context.Context, input GetPodInput) (v1.Pod
 	return *pod, err
 }
 
+// ListCronJobsInput contains all the information needed to list Kubernetes cron jobs
+type ListCronJobsInput struct {
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// ListCronJobs lists Kubernetes cron jobs
+func (k KubernetesClient) ListCronJobs(ctx context.Context, input ListCronJobsInput) ([]batchv1.CronJob, error) {
+	listOptions := metav1.ListOptions{}
+	if input.LabelSelector != "" {
+		listOptions.LabelSelector = input.LabelSelector
+	}
+
+	cronJobs, err := k.Client.BatchV1().CronJobs(input.Namespace).List(ctx, listOptions)
+	if err != nil {
+		return []batchv1.CronJob{}, err
+	}
+
+	if cronJobs == nil {
+		return []batchv1.CronJob{}, errors.New("cron jobs is nil")
+	}
+
+	return cronJobs.Items, err
+}
+
 // ListDeploymentsInput contains all the information needed to list Kubernetes deployments
 type ListDeploymentsInput struct {
 	// Namespace is the Kubernetes namespace
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 1da582acc..f5e63dc55 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -76,6 +76,7 @@ type Job struct {
 	DeploymentID     int64
 	Entrypoint       string
 	Env              map[string]string
+	ID               string
 	Image            string
 	ImagePullSecrets string
 	ImageSourceType  string
@@ -84,6 +85,7 @@ type Job struct {
 	Namespace        string
 	ProcessType      string
 	Schedule         string
+	Suffix           string
 	RemoveContainer  bool
 	WorkingDir       string
 }
@@ -172,11 +174,13 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 	labels := map[string]string{
 		"dokku.com/app-name":         input.AppName,
 		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"dokku.com/cron-id":          input.ID,
 		"dokku.com/process-type":     input.ProcessType,
 	}
 	annotations := map[string]string{
 		"dokku.com/app-name":      input.AppName,
 		"dokku.com/builder-type":  input.ImageSourceType,
+		"dokku.com/cron-id":       input.ID,
 		"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
 		"dokku.com/managed":       "true",
 		"dokku.com/process-type":  input.ProcessType,
@@ -195,12 +199,23 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 		})
 	}
 
+	suffix := input.Suffix
+	if suffix == "" {
+		n := 5
+		b := make([]byte, n)
+		if _, err := rand.Read(b); err != nil {
+			panic(err)
+		}
+		suffix = strings.ToLower(fmt.Sprintf("%X", b))
+	}
+	annotations["dokku.com/job-suffix"] = suffix
+
 	job := batchv1.CronJob{
 		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: fmt.Sprintf("%s-%s-", input.AppName, input.ProcessType),
-			Namespace:    input.Namespace,
-			Labels:       labels,
-			Annotations:  annotations,
+			Name:        fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
+			Namespace:   input.Namespace,
+			Labels:      labels,
+			Annotations: annotations,
 		},
 		Spec: batchv1.CronJobSpec{
 			ConcurrencyPolicy:          batchv1.AllowConcurrent,
@@ -535,12 +550,16 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 		})
 	}
 
-	n := 5
-	b := make([]byte, n)
-	if _, err := rand.Read(b); err != nil {
-		panic(err)
+	suffix := input.Suffix
+	if suffix == "" {
+		n := 5
+		b := make([]byte, n)
+		if _, err := rand.Read(b); err != nil {
+			panic(err)
+		}
+		suffix = strings.ToLower(fmt.Sprintf("%X", b))
 	}
-	suffix := strings.ToLower(fmt.Sprintf("%X", b))
+	annotations["dokku.com/job-suffix"] = suffix
 
 	job := batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index f3f34713b..a6a67f56b 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -276,7 +276,32 @@ data:
 	if err != nil {
 		return fmt.Errorf("Error fetching cron entries: %w", err)
 	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
+		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
+		Namespace:     namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing cron jobs: %w", err)
+	}
+
 	for _, cronEntry := range cronEntries {
+		suffix := ""
+		for _, cronJob := range cronJobs {
+			if cronJob.Labels["dokku.com/cron-id"] == cronEntry.ID {
+				var ok bool
+				suffix, ok = cronJob.Annotations["dokku.com/job-suffix"]
+				if !ok {
+					suffix = ""
+				}
+			}
+		}
+
 		words, err := shellquote.Split(cronEntry.Command)
 		if err != nil {
 			return fmt.Errorf("Error parsing cron command: %w", err)
@@ -285,12 +310,14 @@ data:
 			AppName:          appName,
 			Command:          words,
 			Env:              map[string]string{},
+			ID:               cronEntry.ID,
 			Image:            image,
 			ImagePullSecrets: imagePullSecrets,
 			ImageSourceType:  imageSourceType,
 			Namespace:        namespace,
-			ProcessType:      cronEntry.ID,
+			ProcessType:      "cron",
 			Schedule:         cronEntry.Schedule,
+			Suffix:           suffix,
 			WorkingDir:       workingDir,
 		})
 		if err != nil {
@@ -916,8 +943,6 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 			ProcessType: processType,
 			SelectedPod: selectedPod,
 		})
-
-		return nil
 	default:
 		return fmt.Errorf("Unable to attach as the pod is in an unknown state: %s", selectedPod.Status.Phase)
 	}

From c2b78bb2d70a248a5bd62693bff7c26dd4fa5169 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 16:55:09 -0500
Subject: [PATCH 035/113] feat: implement run:list

---
 plugins/scheduler-k3s/Makefile                |  2 +-
 .../scheduler-k3s/src/triggers/triggers.go    |  5 ++
 plugins/scheduler-k3s/triggers.go             | 70 +++++++++++++++++++
 3 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 776266dec..1a89ea0ee 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,5 +1,5 @@
 SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set subcommands/show-kubeconfig
-TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-stop
+TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
 
diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index f0c64098a..ed701b317 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -109,6 +109,11 @@ func main() {
 		}
 
 		err = scheduler_k3s.TriggerSchedulerRun(scheduler, appName, envCount, args)
+	case "scheduler-run-list":
+		scheduler := flag.Arg(0)
+		appName := flag.Arg(1)
+		format := flag.Arg(2)
+		err = scheduler_k3s.TriggerSchedulerRunList(scheduler, appName, format)
 	default:
 		err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
 	}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index a6a67f56b..885505a2b 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -21,6 +22,7 @@ import (
 	"github.com/fatih/color"
 	"github.com/kballard/go-shellquote"
 	"github.com/rancher/wharfie/pkg/registries"
+	"github.com/ryanuber/columnize"
 	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
 	appsv1 "k8s.io/api/apps/v1"
@@ -951,6 +953,74 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	return nil
 }
 
+// TriggerSchedulerRunList lists one-off run pods for a given application
+func TriggerSchedulerRunList(scheduler string, appName string, format string) error {
+	if scheduler != "k3s" {
+		return nil
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+
+	ctx := context.Background()
+	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
+		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
+		Namespace:     namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("Error getting cron jobs: %w", err)
+	}
+
+	type CronJobEntry struct {
+		ID       string `json:"id"`
+		AppName  string `json:"app"`
+		Command  string `json:"command"`
+		Schedule string `json:"schedule"`
+	}
+
+	data := []CronJobEntry{}
+	lines := []string{"ID | Schedule | Command"}
+	for _, cronJob := range cronJobs {
+		command := ""
+		for _, container := range cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers {
+			if container.Name == fmt.Sprintf("%s-cron", appName) {
+				command = strings.Join(container.Args, " ")
+			}
+		}
+
+		cronID, ok := cronJob.Labels["dokku.com/cron-id"]
+		if !ok {
+			common.LogWarn(fmt.Sprintf("Cron job %s does not have a cron ID label", cronJob.Name))
+			continue
+		}
+
+		lines = append(lines, fmt.Sprintf("%s | %s | %s", cronID, cronJob.Spec.Schedule, command))
+		data = append(data, CronJobEntry{
+			ID:       cronID,
+			AppName:  appName,
+			Command:  command,
+			Schedule: cronJob.Spec.Schedule,
+		})
+	}
+
+	if format == "stdout" {
+		result := columnize.SimpleFormat(lines)
+		fmt.Println(result)
+	} else {
+		b, err := json.Marshal(data)
+		if err != nil {
+			return fmt.Errorf("Error marshalling cron jobs: %w", err)
+		}
+		fmt.Println(string(b))
+	}
+
+	return nil
+}
+
 // TriggerSchedulerPostDelete destroys the scheduler-k3s data for a given app container
 func TriggerSchedulerPostDelete(scheduler string, appName string) error {
 	if scheduler != "k3s" {

From cb62ab02f44cbca0d05c56d6e19b58c9ba70e53c Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 17:09:00 -0500
Subject: [PATCH 036/113] fix: add missing post-deploy triggers

---
 plugins/scheduler-k3s/triggers.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 885505a2b..c8bd628f2 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -428,6 +428,27 @@ data:
 		return err
 	}
 
+	common.LogInfo1("Running post-deploy")
+	_, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Args:          []string{appName, "", "", imageTag},
+		CaptureOutput: false,
+		StreamStdio:   true,
+		Trigger:       "core-post-deploy",
+	})
+	if err != nil {
+		return fmt.Errorf("Error running core-post-deploy: %w", err)
+
+	}
+	_, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Args:          []string{appName, "", "", imageTag},
+		CaptureOutput: false,
+		StreamStdio:   true,
+		Trigger:       "post-deploy",
+	})
+	if err != nil {
+		return fmt.Errorf("Error running post-deploy: %w", err)
+	}
+
 	return nil
 }
 

From 7bce908c2d1e822cf69e6226f562b9388b51b4c9 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 17:15:42 -0500
Subject: [PATCH 037/113] fix: properly set the command for herokuish apps

---
 plugins/scheduler-k3s/functions.go | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index bec2feb88..736056e8c 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -153,7 +153,7 @@ func enterPod(ctx context.Context, input EnterPodInput) error {
 func extractStartCommand(input StartCommandInput) string {
 	command := ""
 	if input.ImageSourceType == "herokuish" {
-		command = "/start " + input.ProcessType
+		return "/start " + input.ProcessType
 	}
 
 	startCommandResp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
@@ -162,14 +162,10 @@ func extractStartCommand(input StartCommandInput) string {
 		CaptureOutput: true,
 		StreamStdio:   false,
 	})
-	if err == nil && startCommandResp.ExitCode == 0 {
+	if err == nil && startCommandResp.ExitCode == 0 && len(startCommandResp.Stdout) > 0 {
 		command = startCommandResp.Stdout
 	}
 
-	if input.ImageSourceType == "herokuish" {
-		return command
-	}
-
 	if input.ImageSourceType == "dockerfile" {
 		startCommandDockerfileResp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
 			Trigger:       "config-get",
@@ -177,7 +173,7 @@ func extractStartCommand(input StartCommandInput) string {
 			CaptureOutput: true,
 			StreamStdio:   false,
 		})
-		if err == nil && startCommandDockerfileResp.ExitCode == 0 {
+		if err == nil && startCommandDockerfileResp.ExitCode == 0 && len(startCommandDockerfileResp.Stdout) > 0 {
 			command = startCommandDockerfileResp.Stdout
 		}
 	}

From ba714c189fe8df0fb4d390af436f8f089fbdf009 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 17:29:28 -0500
Subject: [PATCH 038/113] fix: do not parse flags

---
 tests/unit/run_1.bats | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/run_1.bats b/tests/unit/run_1.bats
index 1ab4ee1dd..da7a599b2 100644
--- a/tests/unit/run_1.bats
+++ b/tests/unit/run_1.bats
@@ -29,7 +29,7 @@ teardown() {
   echo "status: $status"
   assert_success
 
-  run /bin/bash -c "dokku --force --quiet run $TEST_APP python -V"
+  run /bin/bash -c "dokku --force --quiet run $TEST_APP -- python -V"
   echo "output: $output"
   echo "status: $status"
   assert_success

From 997f04afae42ea1896bb9d3532b6b850026cc422 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 17:31:28 -0500
Subject: [PATCH 039/113] chore: document that flag parsing should be stopped
 via --

---
 docs/processes/one-off-tasks.md | 16 ++++++++++++----
 tests/unit/git_1.bats           |  2 +-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/docs/processes/one-off-tasks.md b/docs/processes/one-off-tasks.md
index 11419dc94..452b898a7 100644
--- a/docs/processes/one-off-tasks.md
+++ b/docs/processes/one-off-tasks.md
@@ -20,8 +20,9 @@ The `run` command can be used to run a one-off process for a specific command. T
 > New as of 0.25.0, this container will be removed after the process exits.
 
 ```shell
-# runs `ls -lah` in the `/app` directory of the app `node-js-app`
-dokku run node-js-app ls -lah
+# runs `ls` in the `/app` directory of the app `node-js-app`
+#
+dokku run node-js-app ls
 
 # optionally, run can be passed custom environment variables
 dokku run --env "NODE_ENV=development" --env "PATH=/custom/path" node-js-app npm run mytask
@@ -29,6 +30,13 @@ dokku run --env "NODE_ENV=development" --env "PATH=/custom/path" node-js-app npm
 
 One off containers are removed at the end of process execution.
 
+If the custom command uses flags, it must be separated from the `dokku enter` command via `--` (double dashes):
+
+```shell
+# runs `ls -lah` in the `/app` directory of the app `node-js-app`
+dokku run node-js-app -- ls -lah
+```
+
 #### Running Procfile commands
 
 The `run` command can also be used to run a command defined in the app `Procfile`:
@@ -58,7 +66,7 @@ dokku --label=com.example.test-label=value run node-js-app ls -lah
 One-off containers default to interactive mode where possible. To disable this behavior, specify the `--no-tty` flag:
 
 ```shell
-dokku run --no-tty node-js-app ls -lah
+dokku run --no-tty node-js-app -- ls -lah
 ```
 
 ### Running a detached container
@@ -69,7 +77,7 @@ dokku run --no-tty node-js-app ls -lah
 Finally, a container can be run in "detached" mode via the `run:detached` Dokku command. Running a process in detached mode will immediately return a `CONTAINER_ID`. Detached containers are run without a tty and are also removed at the end of process execution.
 
 ```shell
-dokku run:detached node-js-app ls -lah
+dokku run:detached node-js-app -- ls -lah
 # returns the ID of the new container
 ```
 
diff --git a/tests/unit/git_1.bats b/tests/unit/git_1.bats
index 2c24971d7..e92a2332f 100644
--- a/tests/unit/git_1.bats
+++ b/tests/unit/git_1.bats
@@ -98,7 +98,7 @@ teardown() {
   assert_output_contains "objects"
   assert_output_contains "refs"
 
-  run /bin/bash -c "dokku enter $TEST_APP web test -d .git"
+  run /bin/bash -c "dokku enter $TEST_APP web -- test -d .git"
   echo "output: $output"
   echo "status: $status"
   assert_success

From da4771e6e4178f0fa80932d5a81f66c133276f5e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 17:41:42 -0500
Subject: [PATCH 040/113] fix: uninstall in foreground

---
 plugins/scheduler-k3s/helm.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go
index 067bbffd6..8b986f975 100644
--- a/plugins/scheduler-k3s/helm.go
+++ b/plugins/scheduler-k3s/helm.go
@@ -274,6 +274,7 @@ func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 
 func (h *HelmAgent) UninstallChart(releaseName string) error {
 	uninstall := action.NewUninstall(h.Configuration)
+	uninstall.DeletionPropagation = "foreground"
 	_, err := uninstall.Run(releaseName)
 	if err != nil {
 		return fmt.Errorf("Error uninstalling chart: %w", err)

From 8762bede0720982b008cf733e657c7aec2c640be Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 18:42:55 -0500
Subject: [PATCH 041/113] feat: implement healthcheck support

---
 plugins/app-json/appjson.go       | 36 +++++++++++++
 plugins/scheduler-k3s/template.go | 85 +++++++++++++++++++++++++++++++
 plugins/scheduler-k3s/triggers.go | 13 ++++-
 3 files changed, 133 insertions(+), 1 deletion(-)

diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go
index cba952105..5b11c2492 100644
--- a/plugins/app-json/appjson.go
+++ b/plugins/app-json/appjson.go
@@ -65,6 +65,42 @@ type Formation struct {
 	// MaxParallel is the maximum number of processes to start in parallel
 	MaxParallel *int `json:"max_parallel"`
 }
+type Healthcheck struct {
+	Attempts     int32           `json:"attempts,omitempty"`
+	Command      []string        `json:"command,omitempty"`
+	Content      string          `json:"content,omitempty"`
+	HTTPHeaders  []HTTPHeader    `json:"httpHeaders,omitempty"`
+	InitialDelay int32           `json:"initialDelay,omitempty"`
+	Listening    bool            `json:"listening,omitempty"`
+	Name         string          `json:"name,omitempty"`
+	Path         string          `json:"path,omitempty"`
+	Port         int             `json:"port,omitempty"`
+	Scheme       string          `json:"scheme,omitempty"`
+	Timeout      int32           `json:"timeout,omitempty"`
+	Type         HealthcheckType `json:"type,omitempty"`
+	Uptime       int32           `json:"uptime,omitempty"`
+	Wait         int32           `json:"wait,omitempty"`
+	Warn         bool            `json:"warn,omitempty"`
+	OnFailure    *OnFailure      `json:"onFailure,omitempty"`
+}
+
+type HealthcheckType string
+
+const (
+	HealthcheckType_Liveness  HealthcheckType = "liveness"
+	HealthcheckType_Readiness HealthcheckType = "readiness"
+	HealthcheckType_Startup   HealthcheckType = "startup"
+)
+
+type HTTPHeader struct {
+	Name  string `json:"name,omitempty"`
+	Value string `json:"value,omitempty"`
+}
+
+type OnFailure struct {
+	Command []string `json:"command,omitempty"`
+	Url     string   `json:"url,omitempty"`
+}
 
 // Healthcheck is a struct that represents a single healthcheck from an app.json file
 type Healthcheck struct {
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index f5e63dc55..fca23d116 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
 	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
 	orderedmap "github.com/wk8/go-ordered-map/v2"
@@ -45,6 +46,7 @@ type Deployment struct {
 	Image            string
 	ImagePullSecrets string
 	ImageSourceType  string
+	Healthchecks     []appjson.Healthcheck
 	Namespace        string
 	PrimaryPort      int32
 	PortMaps         []PortMap
@@ -402,6 +404,89 @@ func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 		deployment.Spec.Template.Spec.Containers[0].Args = input.Command
 	}
 
+	if len(input.Healthchecks) > 0 {
+		livenessChecks := []corev1.Probe{}
+		readinessChecks := []corev1.Probe{}
+		startupChecks := []corev1.Probe{}
+		uptimeSeconds := []int32{}
+		for _, healthcheck := range input.Healthchecks {
+			probe := corev1.Probe{
+				ProbeHandler:        corev1.ProbeHandler{},
+				InitialDelaySeconds: healthcheck.InitialDelay,
+				PeriodSeconds:       healthcheck.Wait,
+				TimeoutSeconds:      healthcheck.Timeout,
+				FailureThreshold:    healthcheck.Attempts,
+				SuccessThreshold:    int32(1),
+			}
+			if len(healthcheck.Command) > 0 {
+				probe.ProbeHandler.Exec = &corev1.ExecAction{
+					Command: healthcheck.Command,
+				}
+			} else if healthcheck.Listening {
+				probe.ProbeHandler.TCPSocket = &corev1.TCPSocketAction{
+					Port: intstr.FromInt32(input.PrimaryPort),
+				}
+				for _, header := range healthcheck.HTTPHeaders {
+					if header.Name == "Host" {
+						probe.ProbeHandler.TCPSocket.Host = header.Value
+					}
+				}
+			} else if healthcheck.Path != "" {
+				probe.ProbeHandler.HTTPGet = &corev1.HTTPGetAction{
+					Path:        healthcheck.Path,
+					Port:        intstr.FromInt32(input.PrimaryPort),
+					HTTPHeaders: []corev1.HTTPHeader{},
+				}
+
+				if healthcheck.Scheme != "" {
+					probe.ProbeHandler.HTTPGet.Scheme = corev1.URIScheme(strings.ToUpper(healthcheck.Scheme))
+				}
+
+				for _, header := range healthcheck.HTTPHeaders {
+					probe.ProbeHandler.HTTPGet.HTTPHeaders = append(probe.ProbeHandler.HTTPGet.HTTPHeaders, corev1.HTTPHeader{
+						Name:  header.Name,
+						Value: header.Value,
+					})
+				}
+			} else if healthcheck.Uptime > 0 {
+				uptimeSeconds = append(uptimeSeconds, healthcheck.Uptime)
+			}
+
+			if healthcheck.Type == appjson.HealthcheckType_Liveness {
+				livenessChecks = append(livenessChecks, probe)
+			} else if healthcheck.Type == appjson.HealthcheckType_Readiness {
+				readinessChecks = append(readinessChecks, probe)
+			} else if healthcheck.Type == appjson.HealthcheckType_Startup {
+				startupChecks = append(startupChecks, probe)
+			}
+		}
+		if len(livenessChecks) > 1 {
+			common.LogWarn("Multiple liveness checks are not supported, only the first one will be used")
+		}
+		if len(readinessChecks) > 1 {
+			common.LogWarn("Multiple readiness checks are not supported, only the first one will be used")
+		}
+		if len(startupChecks) > 1 {
+			common.LogWarn("Multiple startup checks are not supported, only the first one will be used")
+		}
+		if len(uptimeSeconds) > 1 {
+			common.LogWarn("Multiple uptime checks are not supported, only the first one will be used")
+		}
+
+		if len(livenessChecks) > 0 {
+			deployment.Spec.Template.Spec.Containers[0].LivenessProbe = &livenessChecks[0]
+		}
+		if len(readinessChecks) > 0 {
+			deployment.Spec.Template.Spec.Containers[0].ReadinessProbe = &readinessChecks[0]
+		}
+		if len(startupChecks) > 0 {
+			deployment.Spec.Template.Spec.Containers[0].StartupProbe = &startupChecks[0]
+		}
+		if len(uptimeSeconds) > 0 {
+			deployment.Spec.MinReadySeconds = uptimeSeconds[0]
+		}
+	}
+
 	if input.ProcessType == "web" {
 		for _, portMap := range input.PortMaps {
 			protocol := "TCP"
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index c8bd628f2..2d4d751cb 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -16,6 +16,7 @@ import (
 	"syscall"
 	"time"
 
+	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
 	"github.com/dokku/dokku/plugins/config"
 	"github.com/dokku/dokku/plugins/cron"
@@ -221,6 +222,11 @@ data:
 		}
 	}
 
+	appJSON, err := appjson.GetAppJSON(appName)
+	if err != nil {
+		return fmt.Errorf("Error getting app.json for deployment: %w", err)
+	}
+
 	workingDir := common.GetWorkingDir(appName, image)
 	deployments := map[string]appsv1.Deployment{}
 	i := 0
@@ -239,7 +245,11 @@ data:
 		i++
 		replicaCountPlaceholder := int32(i * 1000)
 
-		// todo: implement healthchecks
+		healthchecks, ok := appJSON.Healthchecks[processType]
+		if !ok {
+			healthchecks = []appjson.Healthcheck{}
+		}
+
 		// todo: implement deployment annotations
 		// todo: implement pod annotations
 		// todo: implement volumes
@@ -249,6 +259,7 @@ data:
 			Image:            image,
 			ImagePullSecrets: imagePullSecrets,
 			ImageSourceType:  imageSourceType,
+			Healthchecks:     healthchecks,
 			Namespace:        namespace,
 			PrimaryPort:      primaryPort,
 			PortMaps:         portMaps,

From 599692446eacc09e819c79480876aca61f21336c Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 18:46:50 -0500
Subject: [PATCH 042/113] docs: update what is supported

---
 docs/deployment/schedulers/k3s.md | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index c395a606a..acaacdded 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -164,21 +164,25 @@ The following sections describe implemented and unimplemented scheduler function
 
 This plugin implements various functionality through `plugn` triggers to integrate with Docker for running apps on a single server. The following functionality is supported by the `scheduler-k3s` plugin.
 
+- `apps:clone`
 - `apps:destroy`
+- `apps:rename`
 - `cron`
+- `enter`
+- `deploy`
+- healthchecks
+       - Due to Kubernetes limitations, only a single healthcheck is supported for each of the `liveness`, `readiness`, and `startup` healthchecks
+       - Due to Kubernetes limitations, content checks are not supported
+       - Ports specified in the `app.json` are ignored in favor of the container port on the port mapping detected
+- `logs`
+- `ps:stop`
 - `run`
        - The `scheduler-post-run` trigger is not always triggered
 - `run:detached`
-- `enter`
-- `deploy`
-- `logs`
-- `ps:stop`
+- `run:list`
 
 ### Unimplemented command functionality
 
-- `apps:clone`
-- `apps:rename`
-- `run:list`
 - `run:logs`
 - `ps:inspect`
 
@@ -186,7 +190,6 @@ The following Dokku functionality is not implemented at this time.
 
 - `vector` log integration
 - persistent storage
-- healthchecks
 
 ### Logging support
 

From a90f6bdee809d32b8fec574d5d22a3ea9ba06250 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 18:50:32 -0500
Subject: [PATCH 043/113] docs: remove need to add --

This is done transparently now by Dokku, so no need to have a bc-break
---
 docs/processes/one-off-tasks.md | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/docs/processes/one-off-tasks.md b/docs/processes/one-off-tasks.md
index 452b898a7..60ab9844a 100644
--- a/docs/processes/one-off-tasks.md
+++ b/docs/processes/one-off-tasks.md
@@ -30,13 +30,6 @@ dokku run --env "NODE_ENV=development" --env "PATH=/custom/path" node-js-app npm
 
 One off containers are removed at the end of process execution.
 
-If the custom command uses flags, it must be separated from the `dokku enter` command via `--` (double dashes):
-
-```shell
-# runs `ls -lah` in the `/app` directory of the app `node-js-app`
-dokku run node-js-app -- ls -lah
-```
-
 #### Running Procfile commands
 
 The `run` command can also be used to run a command defined in the app `Procfile`:
@@ -66,7 +59,7 @@ dokku --label=com.example.test-label=value run node-js-app ls -lah
 One-off containers default to interactive mode where possible. To disable this behavior, specify the `--no-tty` flag:
 
 ```shell
-dokku run --no-tty node-js-app -- ls -lah
+dokku run --no-tty node-js-app ls -lah
 ```
 
 ### Running a detached container
@@ -77,7 +70,7 @@ dokku run --no-tty node-js-app -- ls -lah
 Finally, a container can be run in "detached" mode via the `run:detached` Dokku command. Running a process in detached mode will immediately return a `CONTAINER_ID`. Detached containers are run without a tty and are also removed at the end of process execution.
 
 ```shell
-dokku run:detached node-js-app -- ls -lah
+dokku run:detached node-js-app ls -lah
 # returns the ID of the new container
 ```
 

From e13561ec4da30e45a517035144c2ba18912de25a Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 18:55:03 -0500
Subject: [PATCH 044/113] refactor: move registry path to const

---
 plugins/scheduler-k3s/scheduler_k3s.go | 2 ++
 plugins/scheduler-k3s/subcommands.go   | 4 ++--
 plugins/scheduler-k3s/triggers.go      | 3 +--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 02462ad14..715d37fac 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -32,6 +32,8 @@ var (
 
 const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
 
+const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml"
+
 var (
 	runtimeScheme  = runtime.NewScheme()
 	codecs         = serializer.NewCodecFactory(runtimeScheme)
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index e149d6a87..9d5550e9f 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -76,7 +76,7 @@ func CommandInitialize() error {
 		return fmt.Errorf("Error installing k3s: %w", installerCmd.ExitError)
 	}
 
-	if err := common.TouchFile("/etc/rancher/k3s/registries.yaml"); err != nil {
+	if err := common.TouchFile(RegistryConfigPath); err != nil {
 		return fmt.Errorf("Error creating initial registries.yaml file")
 	}
 
@@ -84,7 +84,7 @@ func CommandInitialize() error {
 		"setfacl",
 		"-m",
 		"user:dokku:rwx",
-		"/etc/rancher/k3s/registries.yaml",
+		RegistryConfigPath,
 	}, " "))
 	if !registryAclCmd.Execute() {
 		return fmt.Errorf("Error updating acls on k3s registries.yaml file: %w", registryAclCmd.ExitError)
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 2d4d751cb..81ebff23d 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -85,8 +85,7 @@ func TriggerPostRegistryLogin(server string, username string, password string) e
 	}
 
 	registry := registries.Registry{}
-	registryFile := "/etc/rancher/k3s/registries.yaml"
-	yamlFile, err := os.ReadFile(registryFile)
+	yamlFile, err := os.ReadFile(RegistryConfigPath)
 	if err != nil {
 		return fmt.Errorf("Unable to read existing registries.yaml: %w", err)
 	}

From 5449c35dbe459daf914430962047752d1edae8d0 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 19:02:46 -0500
Subject: [PATCH 045/113] fix: correct path to registry config and update
 go.mod

---
 plugins/scheduler-k3s/go.mod      | 4 ++--
 plugins/scheduler-k3s/triggers.go | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 677660577..837b2e2fc 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -3,6 +3,7 @@ module github.com/dokku/dokku/plugins/scheduler-k3s
 go 1.21
 
 require (
+	github.com/dokku/dokku/plugins/app-json v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
@@ -10,6 +11,7 @@ require (
 	github.com/go-resty/resty/v2 v2.11.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/rancher/wharfie v0.6.5
+	github.com/ryanuber/columnize v2.1.2+incompatible
 	github.com/spf13/pflag v1.0.5
 	github.com/traefik/traefik/v2 v2.10.7
 	github.com/wk8/go-ordered-map/v2 v2.1.8
@@ -55,7 +57,6 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dokku/dokku/plugins/app-json v0.0.0-00010101000000-000000000000 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
@@ -129,7 +130,6 @@ require (
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rubenv/sql-migrate v1.5.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/ryanuber/columnize v2.1.2+incompatible // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 81ebff23d..aec94eec7 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -114,7 +114,7 @@ func TriggerPostRegistryLogin(server string, username string, password string) e
 		return fmt.Errorf("Unable to marshal registry configuration to yaml: %w", err)
 	}
 
-	if err := os.WriteFile(registryFile, data, os.FileMode(0644)); err != nil {
+	if err := os.WriteFile(RegistryConfigPath, data, os.FileMode(0644)); err != nil {
 		return fmt.Errorf("Unable to write registry configuration to file: %w", err)
 	}
 

From 27b82f77a3766eebd3cd0d5a334a8cb86f097459 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 21:57:01 -0500
Subject: [PATCH 046/113] fix: drop duplicate

---
 plugins/app-json/appjson.go | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/plugins/app-json/appjson.go b/plugins/app-json/appjson.go
index 5b11c2492..cba952105 100644
--- a/plugins/app-json/appjson.go
+++ b/plugins/app-json/appjson.go
@@ -65,42 +65,6 @@ type Formation struct {
 	// MaxParallel is the maximum number of processes to start in parallel
 	MaxParallel *int `json:"max_parallel"`
 }
-type Healthcheck struct {
-	Attempts     int32           `json:"attempts,omitempty"`
-	Command      []string        `json:"command,omitempty"`
-	Content      string          `json:"content,omitempty"`
-	HTTPHeaders  []HTTPHeader    `json:"httpHeaders,omitempty"`
-	InitialDelay int32           `json:"initialDelay,omitempty"`
-	Listening    bool            `json:"listening,omitempty"`
-	Name         string          `json:"name,omitempty"`
-	Path         string          `json:"path,omitempty"`
-	Port         int             `json:"port,omitempty"`
-	Scheme       string          `json:"scheme,omitempty"`
-	Timeout      int32           `json:"timeout,omitempty"`
-	Type         HealthcheckType `json:"type,omitempty"`
-	Uptime       int32           `json:"uptime,omitempty"`
-	Wait         int32           `json:"wait,omitempty"`
-	Warn         bool            `json:"warn,omitempty"`
-	OnFailure    *OnFailure      `json:"onFailure,omitempty"`
-}
-
-type HealthcheckType string
-
-const (
-	HealthcheckType_Liveness  HealthcheckType = "liveness"
-	HealthcheckType_Readiness HealthcheckType = "readiness"
-	HealthcheckType_Startup   HealthcheckType = "startup"
-)
-
-type HTTPHeader struct {
-	Name  string `json:"name,omitempty"`
-	Value string `json:"value,omitempty"`
-}
-
-type OnFailure struct {
-	Command []string `json:"command,omitempty"`
-	Url     string   `json:"url,omitempty"`
-}
 
 // Healthcheck is a struct that represents a single healthcheck from an app.json file
 type Healthcheck struct {

From 4650585dedff84d73b1282acfd1fa40bb9565b0e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 22:27:07 -0500
Subject: [PATCH 047/113] fix: pull pass from env var

---
 plugins/scheduler-k3s/src/triggers/triggers.go | 3 +--
 plugins/scheduler-k3s/triggers.go              | 4 +++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go
index ed701b317..4e99e14d8 100644
--- a/plugins/scheduler-k3s/src/triggers/triggers.go
+++ b/plugins/scheduler-k3s/src/triggers/triggers.go
@@ -38,8 +38,7 @@ func main() {
 	case "post-registry-login":
 		server := flag.Arg(0)
 		username := flag.Arg(1)
-		password := flag.Arg(2)
-		err = scheduler_k3s.TriggerPostRegistryLogin(server, username, password)
+		err = scheduler_k3s.TriggerPostRegistryLogin(server, username)
 	case "report":
 		appName := flag.Arg(0)
 		err = scheduler_k3s.ReportSingleApp(appName, "", "")
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index aec94eec7..6f902f0e2 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -79,11 +79,13 @@ func TriggerPostDelete(appName string) error {
 
 // TriggerPostRegistryLogin updates the `/etc/rancher/k3s/registries.yaml` to include
 // auth information for the registry. Note that if the file does not exist, it won't be updated.
-func TriggerPostRegistryLogin(server string, username string, password string) error {
+func TriggerPostRegistryLogin(server string, username string) error {
 	if !common.FileExists("/usr/local/bin/k3s") {
 		return nil
 	}
 
+	password := os.Getenv("DOCKER_REGISTRY_PASS")
+
 	registry := registries.Registry{}
 	yamlFile, err := os.ReadFile(RegistryConfigPath)
 	if err != nil {

From 84ef557c9304714a5f2b9597a36dcd94aea7aed4 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 22:27:57 -0500
Subject: [PATCH 048/113] docs: remove now incorrect docs

---
 docs/processes/entering-containers.md | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/docs/processes/entering-containers.md b/docs/processes/entering-containers.md
index 06a3804d2..4d663155d 100644
--- a/docs/processes/entering-containers.md
+++ b/docs/processes/entering-containers.md
@@ -40,10 +40,3 @@ dokku enter node-js-app web echo hi
 # run a long-running command, as one might for a cron task
 dokku enter node-js-app web python script/background-worker.py
 ```
-
-If the custom command uses flags, it must be separated from the `dokku enter` command via `--` (double dashes):
-
-```shell
-# list files in the app
-dokku enter node-js-app web -- ls -lah
-```

From ceae555e432f401717a129ef5970665a471127ef Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 22:28:34 -0500
Subject: [PATCH 049/113] docs: revert docs

---
 docs/processes/one-off-tasks.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/processes/one-off-tasks.md b/docs/processes/one-off-tasks.md
index 60ab9844a..11419dc94 100644
--- a/docs/processes/one-off-tasks.md
+++ b/docs/processes/one-off-tasks.md
@@ -20,9 +20,8 @@ The `run` command can be used to run a one-off process for a specific command. T
 > New as of 0.25.0, this container will be removed after the process exits.
 
 ```shell
-# runs `ls` in the `/app` directory of the app `node-js-app`
-#
-dokku run node-js-app ls
+# runs `ls -lah` in the `/app` directory of the app `node-js-app`
+dokku run node-js-app ls -lah
 
 # optionally, run can be passed custom environment variables
 dokku run --env "NODE_ENV=development" --env "PATH=/custom/path" node-js-app npm run mytask

From dce900588e3ebe62779b3fdd49288510c80895cf Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 18 Jan 2024 22:37:09 -0500
Subject: [PATCH 050/113] chore: revert -- change

---
 tests/unit/git_1.bats | 2 +-
 tests/unit/run_1.bats | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unit/git_1.bats b/tests/unit/git_1.bats
index e92a2332f..2c24971d7 100644
--- a/tests/unit/git_1.bats
+++ b/tests/unit/git_1.bats
@@ -98,7 +98,7 @@ teardown() {
   assert_output_contains "objects"
   assert_output_contains "refs"
 
-  run /bin/bash -c "dokku enter $TEST_APP web -- test -d .git"
+  run /bin/bash -c "dokku enter $TEST_APP web test -d .git"
   echo "output: $output"
   echo "status: $status"
   assert_success
diff --git a/tests/unit/run_1.bats b/tests/unit/run_1.bats
index da7a599b2..1ab4ee1dd 100644
--- a/tests/unit/run_1.bats
+++ b/tests/unit/run_1.bats
@@ -29,7 +29,7 @@ teardown() {
   echo "status: $status"
   assert_success
 
-  run /bin/bash -c "dokku --force --quiet run $TEST_APP -- python -V"
+  run /bin/bash -c "dokku --force --quiet run $TEST_APP python -V"
   echo "output: $output"
   echo "status: $status"
   assert_success

From e6d372f963dcaa3512df984ace962bcc1cef8f15 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 19 Jan 2024 06:27:42 -0500
Subject: [PATCH 051/113] docs: reference image repo templating and git key
 generation for setup

---
 docs/deployment/schedulers/k3s.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index acaacdded..e06117542 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -57,10 +57,10 @@ To ensure images are pushed and pulled from the correct registry, set the correc
 dokku registry:set --global server hub.docker.com
 ```
 
-If using docker hub, you'll need to use a custom repository name. As it is unique, this must be set on a per-app basis.
+If using docker hub, you'll need to use a custom repository name. This can be set via a global template, allowing users access to the app name as the variable `AppName` as shown below.
 
 ```shell
-dokku registry:set node-js-app image-repo my-awesome-prefix/node-js-app
+dokku registry:set --global image-repo-template "my-awesome-prefix/{{ .AppName }}"
 ```
 
 Additionally, apps should be configured to push images on the release phase via the `push-on-release` registry property.
@@ -122,7 +122,13 @@ When attaching an agent or server node, the K3s plugin will look at the IP assoc
 dokku scheduler-k3s:set --global network-interface eth1
 ```
 
-Dokku does not manage the ssh key of the server, but the value that should be added to the remote root user's `/root/.ssh/authorized_keys` file can be checked with the `git:public-key` command:
+Dokku servers may not have an ssh key pair by default, but they can be generated as needed
+
+```shell
+dokku git:generate-deploy-key
+```
+
+This key can then be displayed with the `git:public-key` command:
 
 ```shell
 dokku git:public-key

From cee47b4836f07ac62ef0f25e939f3b6897721e41 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 19 Jan 2024 06:28:04 -0500
Subject: [PATCH 052/113] fix: do not try and override image repo

---
 plugins/scheduler-k3s/triggers.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 6f902f0e2..c343fed03 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -705,8 +705,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	if err != nil {
 		return fmt.Errorf("Error getting running image tag: %w", err)
 	}
-	imageRepo := common.GetAppImageRepo(appName)
-	image, err := common.GetDeployingAppImageName(appName, imageTag, imageRepo)
+	image, err := common.GetDeployingAppImageName(appName, imageTag, "")
 	if err != nil {
 		return fmt.Errorf("Error getting deploying app image name: %w", err)
 	}

From 86da40ba197d4d317fab0467305b2464879c75c1 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 19 Jan 2024 19:21:00 -0500
Subject: [PATCH 053/113] feat: add default-container annotation everywhere

---
 plugins/scheduler-k3s/template.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index fca23d116..f6789ece0 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -212,6 +212,9 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 	}
 	annotations["dokku.com/job-suffix"] = suffix
 
+	podAnnotations := annotations
+	podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)
+
 	job := batchv1.CronJob{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
@@ -236,6 +239,10 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 					BackoffLimit:         ptr.To(int32(0)),
 					PodReplacementPolicy: ptr.To(batchv1.Failed),
 					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels:      labels,
+							Annotations: podAnnotations,
+						},
 						Spec: corev1.PodSpec{
 							Containers: []corev1.Container{
 								{
@@ -353,6 +360,9 @@ func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	}
 	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
 
+	podAnnotations := annotations
+	podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)
+
 	deployment := appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
@@ -369,7 +379,7 @@ func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      labels,
-					Annotations: annotations,
+					Annotations: podAnnotations,
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
@@ -646,6 +656,9 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 	}
 	annotations["dokku.com/job-suffix"] = suffix
 
+	podAnnotations := annotations
+	podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)
+
 	job := batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
@@ -658,7 +671,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      labels,
-					Annotations: annotations,
+					Annotations: podAnnotations,
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{

From 5c4e1dfc39b693db2be8ab80a16aefea99c07b62 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 19 Jan 2024 19:21:17 -0500
Subject: [PATCH 054/113] fix: do not attempt to uninstall a chart that isn't
 available

---
 plugins/scheduler-k3s/helm.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go
index 8b986f975..79bbfe993 100644
--- a/plugins/scheduler-k3s/helm.go
+++ b/plugins/scheduler-k3s/helm.go
@@ -273,9 +273,18 @@ func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 }
 
 func (h *HelmAgent) UninstallChart(releaseName string) error {
+	exists, err := h.ChartExists(releaseName)
+	if err != nil {
+		return fmt.Errorf("Error checking if chart exists: %w", err)
+	}
+
+	if !exists {
+		return nil
+	}
+
 	uninstall := action.NewUninstall(h.Configuration)
 	uninstall.DeletionPropagation = "foreground"
-	_, err := uninstall.Run(releaseName)
+	_, err = uninstall.Run(releaseName)
 	if err != nil {
 		return fmt.Errorf("Error uninstalling chart: %w", err)
 	}

From 9cd06448838eb4f4385bba9a60ae265232f3195e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 19 Jan 2024 19:26:06 -0500
Subject: [PATCH 055/113] refactor: fix issues with pod entering

- ensure we always wait for the specific pod we want to be ready
- enter the default container if none is specified
---
 plugins/scheduler-k3s/functions.go | 94 ++++++++++++++++++++++--------
 plugins/scheduler-k3s/triggers.go  | 30 ++++------
 2 files changed, 80 insertions(+), 44 deletions(-)

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 736056e8c..9fcc3c73e 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/dokku/dokku/plugins/common"
@@ -20,12 +21,12 @@ import (
 )
 
 type EnterPodInput struct {
-	AppName     string
-	Clientset   KubernetesClient
-	Command     []string
-	Entrypoint  string
-	ProcessType string
-	SelectedPod v1.Pod
+	Clientset             KubernetesClient
+	Command               []string
+	Entrypoint            string
+	SelectedContainerName string
+	SelectedPod           v1.Pod
+	WaitTimeout           int
 }
 
 // StartCommandInput contains all the information needed to get the start command
@@ -49,18 +50,20 @@ type StartCommandOutput struct {
 }
 
 type WaitForPodBySelectorRunningInput struct {
-	Clientset KubernetesClient
-	Namespace string
-	Selector  string
-	Timeout   int
-	Waiter    func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
+	Clientset     KubernetesClient
+	Namespace     string
+	LabelSelector string
+	PodName       string
+	Timeout       int
+	Waiter        func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
 }
 
 type WaitForPodToExistInput struct {
-	Clientset  KubernetesClient
-	Namespace  string
-	RetryCount int
-	Selector   string
+	Clientset     KubernetesClient
+	Namespace     string
+	RetryCount    int
+	PodName       string
+	LabelSelector string
 }
 
 func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
@@ -107,13 +110,42 @@ func enterPod(ctx context.Context, input EnterPodInput) error {
 		return fmt.Errorf("Error creating corev1 client: %w", err)
 	}
 
+	labelSelector := []string{}
+	for k, v := range input.SelectedPod.Labels {
+		labelSelector = append(labelSelector, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	if input.WaitTimeout > 0 {
+		input.WaitTimeout = 5
+	}
+
+	err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
+		Clientset:     input.Clientset,
+		Namespace:     input.SelectedPod.Namespace,
+		LabelSelector: strings.Join(labelSelector, ","),
+		PodName:       input.SelectedPod.Name,
+		Timeout:       input.WaitTimeout,
+		Waiter:        isPodReady,
+	})
+	if err != nil {
+		return fmt.Errorf("Error waiting for pod to be ready: %w", err)
+	}
+
+	defaultContainerName, hasDefaultContainer := input.SelectedPod.Annotations["kubectl.kubernetes.io/default-container"]
+	if input.SelectedContainerName == "" && hasDefaultContainer {
+		input.SelectedContainerName = defaultContainerName
+	}
+	if input.SelectedContainerName == "" {
+		return fmt.Errorf("No container specified and no default container found")
+	}
+
 	req := coreclient.RESTClient().Post().
 		Resource("pods").
 		Namespace(input.SelectedPod.Namespace).
 		Name(input.SelectedPod.Name).
 		SubResource("exec")
 
-	req.Param("container", fmt.Sprintf("%s-%s", input.AppName, input.ProcessType))
+	req.Param("container", input.SelectedContainerName)
 	req.Param("stdin", "true")
 	req.Param("stdout", "true")
 	req.Param("stderr", "true")
@@ -270,21 +302,26 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp
 
 func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
 	pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
-		Clientset:  input.Clientset,
-		Namespace:  input.Namespace,
-		RetryCount: 3,
-		Selector:   input.Selector,
+		Clientset:     input.Clientset,
+		LabelSelector: input.LabelSelector,
+		Namespace:     input.Namespace,
+		PodName:       input.PodName,
+		RetryCount:    3,
 	})
 	if err != nil {
 		return fmt.Errorf("Error waiting for pod to exist: %w", err)
 	}
 
 	if len(pods) == 0 {
-		return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.Selector)
+		return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.LabelSelector)
 	}
 
 	timeout := time.Duration(input.Timeout) * time.Second
 	for _, pod := range pods {
+		if input.PodName != "" && pod.Name != input.PodName {
+			break
+		}
+
 		if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
 			print("\n")
 			return fmt.Errorf("Error waiting for pod to be ready: %w", err)
@@ -300,12 +337,21 @@ func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) ([]v1.
 	for i := 0; i < input.RetryCount; i++ {
 		pods, err = input.Clientset.ListPods(ctx, ListPodsInput{
 			Namespace:     input.Namespace,
-			LabelSelector: input.Selector,
+			LabelSelector: input.LabelSelector,
 		})
-		if err == nil {
+		if err != nil {
+			time.Sleep(1 * time.Second)
+		}
+
+		if input.PodName == "" {
 			break
 		}
-		time.Sleep(1 * time.Second)
+
+		for _, pod := range pods {
+			if pod.Name == input.PodName {
+				break
+			}
+		}
 	}
 	if err != nil {
 		return pods, fmt.Errorf("Error listing pods: %w", err)
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index c343fed03..fe4815419 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -181,7 +181,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	}
 	defer os.RemoveAll(chartDir)
 
-	common.LogDebug(fmt.Sprintf("Using chart directory: %s", chartDir))
 	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
 		return fmt.Errorf("Error creating chart templates directory: %w", err)
 	}
@@ -470,7 +469,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		return nil
 	}
 
-	common.LogDebug(fmt.Sprintf("%s %s %s %s", scheduler, appName, processType, podName))
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
@@ -532,11 +530,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		selectedPod = pods[processIndex]
 	}
 
-	processType, ok := selectedPod.Labels["dokku.com/process-type"]
-	if !ok {
-		return fmt.Errorf("Pod %s does not have a process type label", selectedPod.Name)
-	}
-
 	command := args
 	if len(args) == 0 {
 		command = []string{"/bin/bash"}
@@ -554,12 +547,11 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 	}
 
 	return enterPod(ctx, EnterPodInput{
-		AppName:     appName,
 		Clientset:   clientset,
 		Command:     command,
 		Entrypoint:  entrypoint,
-		ProcessType: processType,
 		SelectedPod: selectedPod,
+		WaitTimeout: 10,
 	})
 }
 
@@ -889,10 +881,10 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 
 	batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
 	pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
-		Clientset:  clientset,
-		Namespace:  namespace,
-		RetryCount: 3,
-		Selector:   batchJobSelector,
+		Clientset:     clientset,
+		Namespace:     namespace,
+		RetryCount:    3,
+		LabelSelector: batchJobSelector,
 	})
 	if err != nil {
 		return fmt.Errorf("Error waiting for pod to exist: %w", err)
@@ -907,11 +899,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	}
 
 	err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
-		Clientset: clientset,
-		Namespace: namespace,
-		Selector:  batchJobSelector,
-		Timeout:   300,
-		Waiter:    isPodReady,
+		Clientset:     clientset,
+		Namespace:     namespace,
+		LabelSelector: batchJobSelector,
+		Timeout:       300,
+		Waiter:        isPodReady,
 	})
 	if err != nil {
 		if errors.Is(err, conditions.ErrPodCompleted) {
@@ -970,11 +962,9 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		}
 	case v1.PodRunning:
 		return enterPod(ctx, EnterPodInput{
-			AppName:     appName,
 			Clientset:   clientset,
 			Command:     command,
 			Entrypoint:  entrypoint,
-			ProcessType: processType,
 			SelectedPod: selectedPod,
 		})
 	default:

From 632b26905a2f4f2e0fad39d9a4f31ae1fbced8ba Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Fri, 19 Jan 2024 19:59:44 -0500
Subject: [PATCH 056/113] feat: use common labels instead of custom ones

---
 plugins/scheduler-k3s/template.go | 118 +++++++++++++++---------------
 plugins/scheduler-k3s/triggers.go |  18 ++---
 2 files changed, 68 insertions(+), 68 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index f6789ece0..1b9de2abc 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -174,18 +174,16 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 		return batchv1.CronJob{}, fmt.Errorf("Schedule cannot be empty")
 	}
 	labels := map[string]string{
-		"dokku.com/app-name":         input.AppName,
-		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"app.kubernetes.io/name":     input.ProcessType,
+		"app.kubernetes.io/part-of":  input.AppName,
 		"dokku.com/cron-id":          input.ID,
-		"dokku.com/process-type":     input.ProcessType,
 	}
 	annotations := map[string]string{
-		"dokku.com/app-name":      input.AppName,
-		"dokku.com/builder-type":  input.ImageSourceType,
-		"dokku.com/cron-id":       input.ID,
-		"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
-		"dokku.com/managed":       "true",
-		"dokku.com/process-type":  input.ProcessType,
+		"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
+		"dokku.com/builder-type":    input.ImageSourceType,
+		"dokku.com/cron-id":         input.ID,
+		"dokku.com/managed":         "true",
 	}
 
 	for key, value := range input.Labels {
@@ -347,16 +345,14 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 
 func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	labels := map[string]string{
-		"dokku.com/app-name":         input.AppName,
-		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-		"dokku.com/process-type":     input.ProcessType,
+		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"app.kubernetes.io/name":     input.ProcessType,
+		"app.kubernetes.io/part-of":  input.AppName,
 	}
 	annotations := map[string]string{
-		"dokku.com/app-name":      input.AppName,
-		"dokku.com/builder-type":  input.ImageSourceType,
-		"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
-		"dokku.com/managed":       "true",
-		"dokku.com/process-type":  input.ProcessType,
+		"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
+		"dokku.com/builder-type":    input.ImageSourceType,
+		"dokku.com/managed":         "true",
 	}
 	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
 
@@ -575,19 +571,22 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 		entryPoint = IngressRouteEntrypoint_HTTPS
 	}
 
+	labels := map[string]string{
+		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"app.kubernetes.io/name":     input.ProcessType,
+		"app.kubernetes.io/part-of":  input.AppName,
+	}
+	annotations := map[string]string{
+		"dokku.com/managed": "true",
+	}
+
 	port := fmt.Sprintf("%s-%d-%d", input.PortMap.Scheme, input.PortMap.HostPort, input.PortMap.ContainerPort)
 	ingressRoute := traefikv1alpha1.IngressRoute{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%s", input.ServiceName, port),
-			Namespace: input.Namespace,
-			Labels: map[string]string{
-				"dokku.com/app-name":         input.AppName,
-				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-				"dokku.com/process-type":     input.ProcessType,
-			},
-			Annotations: map[string]string{
-				"dokku.com/managed": "true",
-			},
+			Name:        fmt.Sprintf("%s-%s", input.ServiceName, port),
+			Namespace:   input.Namespace,
+			Labels:      labels,
+			Annotations: annotations,
 		},
 		Spec: traefikv1alpha1.IngressRouteSpec{
 			EntryPoints: []string{string(entryPoint)},
@@ -620,16 +619,14 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 
 func templateKubernetesJob(input Job) (batchv1.Job, error) {
 	labels := map[string]string{
-		"dokku.com/app-name":         input.AppName,
-		"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-		"dokku.com/process-type":     input.ProcessType,
+		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
+		"app.kubernetes.io/name":     input.ProcessType,
+		"app.kubernetes.io/part-of":  input.AppName,
 	}
 	annotations := map[string]string{
-		"dokku.com/app-name":      input.AppName,
-		"dokku.com/builder-type":  input.ImageSourceType,
-		"dokku.com/deployment-id": fmt.Sprint(input.DeploymentID),
-		"dokku.com/managed":       "true",
-		"dokku.com/process-type":  input.ProcessType,
+		"app.kubernetes.io/version": fmt.Sprint(input.DeploymentID),
+		"dokku.com/builder-type":    input.ImageSourceType,
+		"dokku.com/managed":         "true",
 	}
 
 	for key, value := range input.Labels {
@@ -773,17 +770,22 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 
 func templateKubernetesSecret(input Secret) corev1.Secret {
 	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
+	labels := map[string]string{
+		"app.kubernetes.io/instance": secretName,
+		"app.kubernetes.io/name":     fmt.Sprintf("%s-env", input.AppName),
+		"app.kubernetes.io/part-of":  input.AppName,
+	}
+
+	annotations := map[string]string{
+		"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
+		"dokku.com/managed":         "true",
+	}
 	secret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      secretName,
-			Namespace: input.Namespace,
-			Labels: map[string]string{
-				"dokku.com/app-name":      input.AppName,
-				"dokku.com/deployment-id": "DEPLOYMENT_ID_QUOTED",
-			},
-			Annotations: map[string]string{
-				"dokku.com/managed": "true",
-			},
+			Name:        secretName,
+			Namespace:   input.Namespace,
+			Labels:      labels,
+			Annotations: annotations,
 		},
 		Data: map[string][]byte{},
 	}
@@ -792,25 +794,23 @@ func templateKubernetesSecret(input Secret) corev1.Secret {
 }
 
 func templateKubernetesService(input Service) corev1.Service {
+	labels := map[string]string{
+		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, "web"),
+		"app.kubernetes.io/name":     "web",
+		"app.kubernetes.io/part-of":  input.AppName,
+	}
+	annotations := map[string]string{
+		"dokku.com/managed": "true",
+	}
 	service := corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%s", input.AppName, "web"),
-			Namespace: input.Namespace,
-			Labels: map[string]string{
-				"dokku.com/app-name":         input.AppName,
-				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
-				"dokku.com/process-type":     "web",
-			},
-			Annotations: map[string]string{
-				"dokku.com/managed": "true",
-			},
+			Name:        fmt.Sprintf("%s-%s", input.AppName, "web"),
+			Namespace:   input.Namespace,
+			Labels:      labels,
+			Annotations: annotations,
 		},
 		Spec: corev1.ServiceSpec{
-			Selector: map[string]string{
-				"dokku.com/app-name":         input.AppName,
-				"dokku.com/app-process-type": fmt.Sprintf("%s-%s", input.AppName, "web"),
-				"dokku.com/process-type":     "web",
-			},
+			Selector: labels,
 		},
 	}
 
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index fe4815419..00b40d004 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -296,7 +296,7 @@ data:
 	}
 
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
-		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
 		Namespace:     namespace,
 	})
 	if err != nil {
@@ -487,7 +487,7 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 
-	labelSelector := []string{fmt.Sprintf("dokku.com/app-name=%s", appName)}
+	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 1
 	if processType != "" {
 		parts := strings.SplitN(processType, ".", 2)
@@ -498,7 +498,7 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 				return fmt.Errorf("Error parsing process index: %w", err)
 			}
 		}
-		labelSelector = append(labelSelector, fmt.Sprintf("dokku.com/process-type=%s", processType))
+		labelSelector = append(labelSelector, fmt.Sprintf("app.kubernetes.io/name=%s", processType))
 	}
 
 	pods, err := clientset.ListPods(ctx, ListPodsInput{
@@ -577,7 +577,7 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		cancel()
 	}()
 
-	labelSelector := []string{fmt.Sprintf("dokku.com/app-name=%s", appName)}
+	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 0
 	if processType != "" {
 		parts := strings.SplitN(processType, ".", 2)
@@ -588,7 +588,7 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 				return fmt.Errorf("Error parsing process index: %w", err)
 			}
 		}
-		labelSelector = append(labelSelector, fmt.Sprintf("dokku.com/process-type=%s", processType))
+		labelSelector = append(labelSelector, fmt.Sprintf("app.kubernetes.io/name=%s", processType))
 	}
 
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
@@ -743,7 +743,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	}
 
 	labels := map[string]string{
-		"dokku.com/app-name": appName,
+		"app.kubernetes.io/part-of": appName,
 	}
 
 	if os.Getenv("DOKKU_TRACE") == "1" {
@@ -990,7 +990,7 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er
 
 	ctx := context.Background()
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
-		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
 		Namespace:     namespace,
 	})
 	if err != nil {
@@ -1087,14 +1087,14 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 	deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
 		Namespace:     namespace,
-		LabelSelector: fmt.Sprintf("dokku.com/app-name=%s", appName),
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
 	})
 	if err != nil {
 		return fmt.Errorf("Error listing deployments: %w", err)
 	}
 
 	for _, deployment := range deployments {
-		processType, ok := deployment.Annotations["dokku.com/process-type"]
+		processType, ok := deployment.Annotations["app.kubernetes.io/name"]
 		if !ok {
 			return fmt.Errorf("Deployment %s does not have a process type annotation", deployment.Name)
 		}

From af3009f0dac3ba282466ef61233e3b655f2215d1 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 04:29:31 -0500
Subject: [PATCH 057/113] chore: bump go modules

---
 plugins/scheduler-k3s/go.mod |  9 ++++++---
 plugins/scheduler-k3s/go.sum | 24 ++++++++++++++++++------
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 837b2e2fc..edd7329dc 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -93,6 +93,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.1 // indirect
+	github.com/kr/fs v0.1.0 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.10.9 // indirect
@@ -102,6 +103,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/melbahja/goph v1.4.0 // indirect
 	github.com/miekg/dns v1.1.55 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -123,6 +125,7 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/sftp v1.13.5 // indirect
 	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
@@ -144,13 +147,13 @@ require (
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index e79fd1b0a..bb66402e8 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -248,6 +248,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
 github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -286,6 +288,8 @@ github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/melbahja/goph v1.4.0 h1:z0PgDbBFe66lRYl3v5dGb9aFgPy0kotuQ37QOwSQFqs=
+github.com/melbahja/goph v1.4.0/go.mod h1:uG+VfK2Dlhk+O32zFrRlc3kYKTlV6+BtvPWd/kK7U68=
 github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo=
 github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -345,6 +349,8 @@ github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rK
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
+github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -458,10 +464,12 @@ golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
-golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -484,6 +492,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@@ -512,8 +521,10 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -523,8 +534,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -532,11 +543,12 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=

From 69e951166691635e1bb3f0285c36e2e6204e836f Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 04:58:02 -0500
Subject: [PATCH 058/113] feat: implement cluster joining for server and worker
 nodes

Different options get applied to each depending on what is needed for the role.

This also updates how we initialize the cluster to better support how metrics exposure and cross-server networking.
---
 docs/deployment/schedulers/k3s.md             |  25 +-
 plugins/scheduler-k3s/Makefile                |   2 +-
 plugins/scheduler-k3s/functions.go            |  67 ++-
 plugins/scheduler-k3s/k8s.go                  |  60 +++
 .../src/subcommands/subcommands.go            |  15 +-
 plugins/scheduler-k3s/subcommands.go          | 445 +++++++++++++++++-
 plugins/scheduler-k3s/triggers.go             |   8 +-
 7 files changed, 593 insertions(+), 29 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index e06117542..35d5f7f18 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -145,10 +145,31 @@ In some cases, Dokku must be setup from scratch and joined to an existing cluste
 dokku scheduler-k3s:set --global token topsecret:server:token
 ```
 
-Next, run the `scheduler-k3s:join-cluster` command. This command takes a single server node in the cluster, will SSH onto the specified server, retrieve the kubeconfig, and then configure kubectl on the Dokku node to speak with the specified k3s cluster. Dokku will also periodically retrieve all server nodes in the cluster so that server nodes can be safely replaced as needed.
+Next, run the `scheduler-k3s:cluster-add` command. This command takes a single server node in the cluster, will SSH onto the specified server, retrieve the kubeconfig, and then configure kubectl on the Dokku node to speak with the specified k3s cluster. 
+
+> [!TODO]
+> TODO: Dokku will also periodically retrieve all server nodes in the cluster so that server nodes can be safely replaced as needed.
 
 ```shell
-dokku scheduler-k3s:join-cluster root@server-1.example.com
+dokku scheduler-k3s:cluster-add ssh://root@worker-1.example.com
+```
+
+If the server isn't in the `known_hosts` file, the connection will fail. This can be bypassed by setting the `--insecure-allow-unknown-hosts` flag:
+
+```shell
+dokku scheduler-k3s:cluster-add --insecure-allow-unknown-hosts ssh://root@worker-1.example.com
+```
+
+New server nodes are added as worker nodes which run application workloads but do not otherwise participate in managing the cluster. For high availability, new server nodes can be added to the cluster by specifying the `--role` flag:
+
+```shell
+dokku scheduler-kes:cluster-add --role server ssh://root@cluster-1.example.com
+```
+
+Server nodes allow any workloads to be scheduled on them by default, in addition to the control-plane, etcd, and the scheduler itself. To avoid app workloads being scheduled on your control-plane, use the `--taint-scheduling` flag:
+
+```shell
+dokku scheduler-kes:cluster-add --role server --taint-scheduling ssh://root@cluster-1.example.com
 ```
 
 ### Using kubectl remotely
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 1a89ea0ee..84c1ed2bd 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/initialize subcommands/report subcommands/set subcommands/show-kubeconfig
+SUBCOMMANDS = subcommands/initialize subcommands/join-cluster subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
 TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 9fcc3c73e..2ebf51e7b 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -49,6 +49,13 @@ type StartCommandOutput struct {
 	Command []string
 }
 
+type WaitForNodeToExistInput struct {
+	Clientset  KubernetesClient
+	Namespace  string
+	RetryCount int
+	NodeName   string
+}
+
 type WaitForPodBySelectorRunningInput struct {
 	Clientset     KubernetesClient
 	Namespace     string
@@ -188,25 +195,25 @@ func extractStartCommand(input StartCommandInput) string {
 		return "/start " + input.ProcessType
 	}
 
-	startCommandResp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+	resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
 		Trigger:       "config-get",
 		Args:          []string{input.AppName, "DOKKU_START_CMD"},
 		CaptureOutput: true,
 		StreamStdio:   false,
 	})
-	if err == nil && startCommandResp.ExitCode == 0 && len(startCommandResp.Stdout) > 0 {
-		command = startCommandResp.Stdout
+	if err == nil && resp.ExitCode == 0 && len(resp.Stdout) > 0 {
+		command = strings.TrimSpace(resp.Stdout)
 	}
 
 	if input.ImageSourceType == "dockerfile" {
-		startCommandDockerfileResp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+		resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
 			Trigger:       "config-get",
 			Args:          []string{input.AppName, "DOKKU_DOCKERFILE_START_CMD"},
 			CaptureOutput: true,
 			StreamStdio:   false,
 		})
-		if err == nil && startCommandDockerfileResp.ExitCode == 0 && len(startCommandDockerfileResp.Stdout) > 0 {
-			command = startCommandDockerfileResp.Stdout
+		if err == nil && resp.ExitCode == 0 && len(resp.Stdout) > 0 {
+			command = strings.TrimSpace(resp.Stdout)
 		}
 	}
 
@@ -278,6 +285,22 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
+func isK3sInstalled() error {
+	if !common.FileExists("/usr/local/bin/k3s") {
+		return fmt.Errorf("k3s binary is not available")
+	}
+
+	if !common.FileExists(RegistryConfigPath) {
+		return fmt.Errorf("k3s registry config is not available")
+	}
+
+	if !common.FileExists(KubeConfigPath) {
+		return fmt.Errorf("k3s kubeconfig is not available")
+	}
+
+	return nil
+}
+
 func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
 	return func(ctx context.Context) (bool, error) {
 		fmt.Printf(".") // progress bar!
@@ -331,6 +354,37 @@ func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelector
 	return nil
 }
 
+func waitForNodeToExist(ctx context.Context, input WaitForNodeToExistInput) ([]v1.Node, error) {
+	var matchingNodes []v1.Node
+	var err error
+	for i := 0; i < input.RetryCount; i++ {
+		nodes, err := input.Clientset.ListNodes(ctx, ListNodesInput{})
+		if err != nil {
+			time.Sleep(1 * time.Second)
+		}
+
+		if input.NodeName == "" {
+			matchingNodes = nodes
+			break
+		}
+
+		for _, node := range nodes {
+			if node.Name == input.NodeName {
+				matchingNodes = append(matchingNodes, node)
+				break
+			}
+		}
+		if len(matchingNodes) > 0 {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	if err != nil {
+		return matchingNodes, fmt.Errorf("Error listing nodes: %w", err)
+	}
+	return matchingNodes, nil
+}
+
 func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) ([]v1.Pod, error) {
 	var pods []v1.Pod
 	var err error
@@ -352,6 +406,7 @@ func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) ([]v1.
 				break
 			}
 		}
+		time.Sleep(1 * time.Second)
 	}
 	if err != nil {
 		return pods, fmt.Errorf("Error listing pods: %w", err)
diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go
index c3b5766e9..49accdbeb 100644
--- a/plugins/scheduler-k3s/k8s.go
+++ b/plugins/scheduler-k3s/k8s.go
@@ -3,7 +3,10 @@ package scheduler_k3s
 import (
 	"context"
 	"errors"
+	"fmt"
 
+	"github.com/dokku/dokku/plugins/common"
+	"github.com/go-openapi/jsonpointer"
 	appsv1 "k8s.io/api/apps/v1"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -11,6 +14,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -165,6 +169,37 @@ func (k KubernetesClient) GetPod(ctx context.Context, input GetPodInput) (v1.Pod
 	return *pod, err
 }
 
+// LabelNodeInput contains all the information needed to label a Kubernetes node
+type LabelNodeInput struct {
+	// Name is the Kubernetes node name
+	Name string
+	// Key is the label key
+	Key string
+	// Value is the label value
+	Value string
+}
+
+// LabelNode labels a Kubernetes node
+func (k KubernetesClient) LabelNode(ctx context.Context, input LabelNodeInput) error {
+	node, err := k.Client.CoreV1().Nodes().Get(ctx, input.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if node == nil {
+		return errors.New("node is nil")
+	}
+
+	keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape("kubernetes.io/role"))
+	patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, "worker")
+	_, err = k.Client.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to label node: %w", err)
+	}
+
+	return nil
+}
+
 // ListCronJobsInput contains all the information needed to list Kubernetes cron jobs
 type ListCronJobsInput struct {
 	// LabelSelector is the Kubernetes label selector
@@ -230,6 +265,31 @@ func (k KubernetesClient) ListNamespaces(ctx context.Context) ([]v1.Namespace, e
 	return namespaces.Items, nil
 }
 
+// ListNodesInput contains all the information needed to list Kubernetes nodes
+type ListNodesInput struct {
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+}
+
+// ListNodes lists Kubernetes nodes
+func (k KubernetesClient) ListNodes(ctx context.Context, input ListNodesInput) ([]v1.Node, error) {
+	listOptions := metav1.ListOptions{}
+	if input.LabelSelector != "" {
+		common.LogDebug(fmt.Sprintf("Using label selector: %s", input.LabelSelector))
+		listOptions.LabelSelector = input.LabelSelector
+	}
+	nodeList, err := k.Client.CoreV1().Nodes().List(ctx, listOptions)
+	if err != nil {
+		return []v1.Node{}, err
+	}
+
+	if nodeList == nil {
+		return []v1.Node{}, errors.New("pod list is nil")
+	}
+
+	return nodeList.Items, err
+}
+
 // ListPodsInput contains all the information needed to list Kubernetes pods
 type ListPodsInput struct {
 	// Namespace is the Kubernetes namespace
diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go
index 7915d1fe7..04a9f7dc0 100644
--- a/plugins/scheduler-k3s/src/subcommands/subcommands.go
+++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go
@@ -20,8 +20,17 @@ func main() {
 	switch subcommand {
 	case "initialize":
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
+		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandInitialize()
+		err = scheduler_k3s.CommandInitialize(*taintScheduling)
+	case "cluster-add":
+		args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError)
+		allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts")
+		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
+		role := args.String("role", "worker", "role: [ server | worker ]")
+		args.Parse(os.Args[2:])
+		remoteHost := args.Arg(0)
+		err = scheduler_k3s.CommandClusterAdd(*role, remoteHost, *allowUknownHosts, *taintScheduling)
 	case "report":
 		args := flag.NewFlagSet("scheduler-k3s:report", flag.ExitOnError)
 		format := args.String("format", "stdout", "format: [ stdout | json ]")
@@ -48,6 +57,10 @@ func main() {
 		args := flag.NewFlagSet("scheduler-k3s:show-kubeconfig", flag.ExitOnError)
 		args.Parse(os.Args[2:])
 		err = scheduler_k3s.CommandShowKubeconfig()
+	case "uninstall":
+		args := flag.NewFlagSet("scheduler-k3s:uninstall", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		err = scheduler_k3s.CommandUninstall()
 	default:
 		err = fmt.Errorf("Invalid plugin subcommand call: %s", subcommand)
 	}
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 9d5550e9f..5a013322c 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -1,7 +1,11 @@
 package scheduler_k3s
 
 import (
+	"context"
+	"crypto/rand"
 	"fmt"
+	"net"
+	"net/url"
 	"os"
 	"strings"
 
@@ -10,11 +14,77 @@ import (
 )
 
 // CommandInitialize initializes a k3s cluster on the local server
-func CommandInitialize() error {
-	if common.FileExists("/usr/local/bin/k3s") {
+func CommandInitialize(taintScheduling bool) error {
+	if err := isK3sInstalled(); err == nil {
 		return fmt.Errorf("k3s already installed, cannot re-initialize k3s")
 	}
 
+	networkInterface := common.PropertyGetDefault("scheduler-k3s", "--global", "network-interface", "eth0")
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return fmt.Errorf("Unable to get network interfaces: %w", err)
+	}
+
+	serverIp := ""
+	for _, iface := range ifaces {
+		if iface.Name == networkInterface {
+			addr, err := iface.Addrs()
+			if err != nil {
+				return fmt.Errorf("Unable to get network addresses for interface %s: %w", networkInterface, err)
+			}
+			for _, a := range addr {
+				if ipnet, ok := a.(*net.IPNet); ok {
+					if ipnet.IP.To4() != nil {
+						serverIp = ipnet.IP.String()
+					}
+				}
+			}
+		}
+	}
+
+	if len(serverIp) == 0 {
+		return fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface))
+	}
+
+	common.LogInfo1Quiet("Initializing k3s")
+
+	common.LogInfo2Quiet("Updating apt")
+	aptUpdateCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: "apt-get",
+		Args: []string{
+			"update",
+		},
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call apt-get update command: %w", err)
+	}
+	if aptUpdateCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from apt-get update command: %d", aptUpdateCmd.ExitCode)
+	}
+
+	common.LogInfo2Quiet("Installing k3s dependencies")
+	aptInstallCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: "apt-get",
+		Args: []string{
+			"-y",
+			"install",
+			"ca-certificates",
+			"curl",
+			"open-iscsi",
+			"nfs-common",
+			"wireguard",
+		},
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call apt-get install command: %w", err)
+	}
+	if aptInstallCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from apt-get install command: %d", aptInstallCmd.ExitCode)
+	}
+
+	common.LogInfo2Quiet("Downloading k3s installer")
 	client := resty.New()
 	resp, err := client.R().
 		Get("https://get.k3s.io")
@@ -63,33 +133,358 @@ func CommandInitialize() error {
 		return fmt.Errorf("Unable to set k3s token: %w", err)
 	}
 
-	installerCmd := common.NewShellCmd(strings.Join([]string{
-		f.Name(),
+	nodeName := serverIp
+	n := 5
+	b := make([]byte, n)
+	if _, err := rand.Read(b); err != nil {
+		return fmt.Errorf("Unable to generate random node name: %w", err)
+	}
+	nodeName = strings.ReplaceAll(strings.ToLower(fmt.Sprintf("ip-%s-%s", nodeName, fmt.Sprintf("%X", b))), ".", "-")
+
+	args := []string{
 		// initialize the cluster
 		"--cluster-init",
+		// disable local-storage
+		"--disable", "local-storage",
+		// expose etcd metrics
+		"--etcd-expose-metrics",
+		// use wireguard for flannel
+		"--flannel-backend=wireguard-native",
+		// bind controller-manager to all interfaces
+		"--kube-controller-manager-arg", "bind-address=0.0.0.0",
+		// bind proxy metrics to all interfaces
+		"--kube-proxy-arg", "metrics-bind-address=0.0.0.0",
+		// bind scheduler to all interfaces
+		"--kube-scheduler-arg", "bind-address=0.0.0.0",
+		// gc terminated pods
+		"--kube-controller-manager-arg", "terminated-pod-gc-threshold=10",
+		// specify the node name
+		"--node-name", nodeName,
 		// allow access for the dokku user
 		"--write-kubeconfig-mode", "0644",
 		// specify a token
 		"--token", token,
-	}, " "))
-	if !installerCmd.Execute() {
-		return fmt.Errorf("Error installing k3s: %w", installerCmd.ExitError)
+	}
+	if taintScheduling {
+		args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
+	}
+
+	common.LogInfo2Quiet("Running k3s installer")
+	installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command:     f.Name(),
+		Args:        args,
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call k3s installer command: %w", err)
+	}
+	if installerCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from k3s installer command: %d", installerCmd.ExitCode)
 	}
 
 	if err := common.TouchFile(RegistryConfigPath); err != nil {
 		return fmt.Errorf("Error creating initial registries.yaml file")
 	}
 
-	registryAclCmd := common.NewShellCmd(strings.Join([]string{
-		"setfacl",
-		"-m",
-		"user:dokku:rwx",
-		RegistryConfigPath,
-	}, " "))
-	if !registryAclCmd.Execute() {
-		return fmt.Errorf("Error updating acls on k3s registries.yaml file: %w", registryAclCmd.ExitError)
+	common.LogInfo2Quiet("Setting registries.yaml permissions")
+	registryAclCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: "setfacl",
+		Args: []string{
+			"-m",
+			"user:dokku:rwx",
+			RegistryConfigPath,
+		},
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call setfacl command: %w", err)
+	}
+	if registryAclCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from setfacl command: %d", registryAclCmd.ExitCode)
 	}
 
+	common.LogInfo2Quiet("Installing k3s automatic upgrader")
+	upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: "kubectl",
+		Args: []string{
+			"apply",
+			"-f",
+			"https://github.com/rancher/system-upgrade-controller/releases/latest/download/system-upgrade-controller.yaml",
+		},
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call kubectl command: %w", err)
+	}
+	if upgradeCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from kubectl command: %d", upgradeCmd.ExitCode)
+	}
+
+	common.LogVerboseQuiet("Done")
+
+	return nil
+}
+
+// CommandClusterAdd adds a server to the k3s cluster
+func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, taintScheduling bool) error {
+	if err := isK3sInstalled(); err != nil {
+		return fmt.Errorf("k3s not installed, cannot join cluster")
+	}
+
+	if role != "server" && role != "worker" {
+		return fmt.Errorf("Invalid server-type: %s", role)
+	}
+
+	token := common.PropertyGet("scheduler-k3s", "--global", "token")
+	if len(token) == 0 {
+		return fmt.Errorf("Missing k3s token")
+	}
+
+	if taintScheduling && role == "worker" {
+		return fmt.Errorf("Taint scheduling can only be used on the server role")
+	}
+
+	networkInterface := common.PropertyGetDefault("scheduler-k3s", "--global", "network-interface", "eth0")
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return fmt.Errorf("Unable to get network interfaces: %w", err)
+	}
+
+	serverIp := ""
+	for _, iface := range ifaces {
+		if iface.Name == networkInterface {
+			addr, err := iface.Addrs()
+			if err != nil {
+				return fmt.Errorf("Unable to get network addresses for interface %s: %w", networkInterface, err)
+			}
+			for _, a := range addr {
+				if ipnet, ok := a.(*net.IPNet); ok {
+					if ipnet.IP.To4() != nil {
+						serverIp = ipnet.IP.String()
+					}
+				}
+			}
+		}
+	}
+
+	if len(serverIp) == 0 {
+		return fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface))
+	}
+
+	// todo: check if k3s is installed on the remote host
+
+	k3sVersionCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: "k3s",
+		Args: []string{
+			"--version",
+		},
+		CaptureOutput: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call k3s version command: %w", err)
+	}
+	if k3sVersionCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from k3s --version command: %d", k3sVersionCmd.ExitCode)
+	}
+
+	k3sVersion := ""
+	k3sVersionLines := strings.Split(string(k3sVersionCmd.Stdout), "\n")
+	if len(k3sVersionLines) > 0 {
+		k3sVersionParts := strings.Split(k3sVersionLines[0], " ")
+		if len(k3sVersionParts) != 4 {
+			return fmt.Errorf("Unable to get k3s version from k3s --version: %s", k3sVersionCmd.Stdout)
+		}
+		k3sVersion = k3sVersionParts[2]
+	}
+	common.LogDebug(fmt.Sprintf("k3s version: %s", k3sVersion))
+
+	common.LogInfo1(fmt.Sprintf("Joining %s to k3s cluster as %s", remoteHost, role))
+	common.LogInfo2Quiet("Updating apt")
+	aptUpdateCmd, err := common.CallSshCommand(common.SshCommandInput{
+		Command: "apt-get",
+		Args: []string{
+			"update",
+		},
+		AllowUknownHosts: allowUknownHosts,
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+		Sudo:             true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call apt-get update command over ssh: %w", err)
+	}
+	if aptUpdateCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from apt-get update command over ssh: %d", aptUpdateCmd.ExitCode)
+	}
+
+	common.LogInfo2Quiet("Installing k3s dependencies")
+	aptInstallCmd, err := common.CallSshCommand(common.SshCommandInput{
+		Command: "apt-get",
+		Args: []string{
+			"-y",
+			"install",
+			"ca-certificates",
+			"curl",
+			"open-iscsi",
+			"nfs-common",
+			"wireguard",
+		},
+		AllowUknownHosts: allowUknownHosts,
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+		Sudo:             true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call apt-get install command over ssh: %w", err)
+	}
+	if aptInstallCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from apt-get install command over ssh: %d", aptInstallCmd.ExitCode)
+	}
+
+	common.LogInfo2Quiet("Downloading k3s installer")
+	curlTask, err := common.CallSshCommand(common.SshCommandInput{
+		Command: "curl",
+		Args: []string{
+			"-o /tmp/k3s-installer.sh",
+			"https://get.k3s.io",
+		},
+		AllowUknownHosts: allowUknownHosts,
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call curl command over ssh: %w", err)
+	}
+	if curlTask.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from curl command over ssh: %d", curlTask.ExitCode)
+	}
+
+	common.LogInfo2Quiet("Setting k3s installer permissions")
+	chmodCmd, err := common.CallSshCommand(common.SshCommandInput{
+		Command: "chmod",
+		Args: []string{
+			"0755",
+			"/tmp/k3s-installer.sh",
+		},
+		AllowUknownHosts: allowUknownHosts,
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call chmod command over ssh: %w", err)
+	}
+	if chmodCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
+	}
+
+	u, err := url.Parse(remoteHost)
+	if err != nil {
+		return fmt.Errorf("failed to parse remote host: %w", err)
+	}
+
+	nodeName := u.Hostname()
+	n := 5
+	b := make([]byte, n)
+	if _, err := rand.Read(b); err != nil {
+		return fmt.Errorf("Unable to generate random node name: %w", err)
+	}
+	nodeName = strings.ReplaceAll(strings.ToLower(fmt.Sprintf("ip-%s-%s", nodeName, fmt.Sprintf("%X", b))), ".", "-")
+
+	args := []string{
+		// disable local-storage
+		"--disable", "local-storage",
+		// add a node label
+		"--node-label=node_type=worker",
+		// use wireguard for flannel
+		"--flannel-backend=wireguard-native",
+		// specify the node name
+		"--node-name", nodeName,
+		// server to connect to as the main
+		"--server",
+		fmt.Sprintf("https://%s:6443", serverIp),
+		// specify a token
+		"--token",
+		token,
+	}
+
+	if role == "server" {
+		args = append([]string{"server"}, args...)
+		// expose etcd metrics
+		args = append(args, "--etcd-expose-metrics")
+		// bind controller-manager to all interfaces
+		args = append(args, "--kube-controller-manager-arg", "bind-address=0.0.0.0")
+		// bind proxy metrics to all interfaces
+		args = append(args, "--kube-proxy-arg", "metrics-bind-address=0.0.0.0")
+		// bind scheduler to all interfaces
+		args = append(args, "--kube-scheduler-arg", "bind-address=0.0.0.0")
+		// gc terminated pods
+		args = append(args, "--kube-controller-manager-arg", "terminated-pod-gc-threshold=10")
+		// allow access for the dokku user
+		args = append(args, "--write-kubeconfig-mode", "0644")
+	} else {
+		// disable etcd on workers
+		args = append(args, "--disable-etcd")
+		// disable apiserver on workers
+		args = append(args, "--disable-apiserver")
+		// disable controller-manager on workers
+		args = append(args, "--disable-controller-manager")
+		// disable scheduler on workers
+		args = append(args, "--disable-scheduler")
+		// bind proxy metrics to all interfaces
+		args = append(args, "--kube-proxy-arg", "metrics-bind-address=0.0.0.0")
+	}
+	if taintScheduling {
+		args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
+	}
+
+	common.LogInfo2Quiet("Joining k3s cluster")
+	joinCmd, err := common.CallSshCommand(common.SshCommandInput{
+		Command:          "/tmp/k3s-installer.sh",
+		Args:             args,
+		AllowUknownHosts: allowUknownHosts,
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+		Sudo:             true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call k3s installer command over ssh: %w", err)
+	}
+	if joinCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
+	}
+
+	if role == "worker" {
+		ctx := context.Background()
+		clientset, err := NewKubernetesClient()
+		if err != nil {
+			return fmt.Errorf("Unable to create kubernetes client: %w", err)
+		}
+
+		common.LogInfo2Quiet("Waiting for node to exist")
+		nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
+			Clientset:  clientset,
+			NodeName:   nodeName,
+			RetryCount: 20,
+		})
+		if err != nil {
+			return fmt.Errorf("Error waiting for pod to exist: %w", err)
+		}
+		if len(nodes) == 0 {
+			return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled kubernetes.io/role=worker")
+		}
+
+		common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker")
+		err = clientset.LabelNode(ctx, LabelNodeInput{
+			Name:  nodes[0].Name,
+			Key:   "kubernetes.io/role",
+			Value: "worker",
+		})
+		if err != nil {
+			return fmt.Errorf("Unable to patch node: %w", err)
+		}
+	}
+
+	common.LogVerboseQuiet("Done")
 	return nil
 }
 
@@ -132,3 +527,23 @@ func CommandShowKubeconfig() error {
 
 	return nil
 }
+
+func CommandUninstall() error {
+	if err := isK3sInstalled(); err != nil {
+		return fmt.Errorf("k3s not installed, cannot uninstall")
+	}
+
+	common.LogInfo1("Uninstalling k3s")
+	uninstallerCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command:     "/usr/local/bin/k3s-uninstall.sh",
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call k3s uninstaller command: %w", err)
+	}
+	if uninstallerCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from k3s uninstaller command: %d", uninstallerCmd.ExitCode)
+	}
+
+	return nil
+}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 00b40d004..5cd66fd70 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -713,24 +713,24 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 
 	dokkuRmContainer := os.Getenv("DOKKU_RM_CONTAINER")
 	if dokkuRmContainer == "" {
-		appRmContainer, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+		resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
 			Trigger:       "config-get",
 			Args:          []string{appName, "DOKKU_RM_CONTAINER"},
 			CaptureOutput: true,
 			StreamStdio:   false,
 		})
 		if err != nil {
-			globalRmContainer, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
+			resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
 				Trigger:       "config-get-global",
 				Args:          []string{"DOKKU_RM_CONTAINER"},
 				CaptureOutput: true,
 				StreamStdio:   false,
 			})
 			if err == nil {
-				dokkuRmContainer = globalRmContainer.Stdout
+				dokkuRmContainer = strings.TrimSpace(resp.Stdout)
 			}
 		} else {
-			dokkuRmContainer = appRmContainer.Stdout
+			dokkuRmContainer = strings.TrimSpace(resp.Stdout)
 		}
 	}
 	if dokkuRmContainer == "" {

From a1428ceb7490acfbbf9b54ab1975b272137c4d08 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 04:59:15 -0500
Subject: [PATCH 059/113] fix: do not set a static token and generate one as
 needed

---
 plugins/scheduler-k3s/subcommands.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 5a013322c..6bc94d701 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -127,10 +127,18 @@ func CommandInitialize(taintScheduling bool) error {
 		return fmt.Errorf("Invalid k3s installer filesize")
 	}
 
-	// todo: allow this to be passed as an option or environment variable
-	token := "password"
-	if err := CommandSet("--global", "token", token); err != nil {
-		return fmt.Errorf("Unable to set k3s token: %w", err)
+	token := common.PropertyGet("scheduler-k3s", "--global", "token")
+	if len(token) == 0 {
+		n := 5
+		b := make([]byte, n)
+		if _, err := rand.Read(b); err != nil {
+			return fmt.Errorf("Unable to generate random node name: %w", err)
+		}
+
+		token := strings.ToLower(fmt.Sprintf("%X", b))
+		if err := CommandSet("--global", "token", token); err != nil {
+			return fmt.Errorf("Unable to set k3s token: %w", err)
+		}
 	}
 
 	nodeName := serverIp

From b85bc49f03eb4cf3eadf829a8455e1565fc5d2d5 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 04:59:23 -0500
Subject: [PATCH 060/113] docs: update k3s documentation

---
 docs/deployment/schedulers/k3s.md | 75 ++++++++++---------------------
 1 file changed, 23 insertions(+), 52 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 35d5f7f18..0c93c377e 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -91,30 +91,10 @@ At this point, all app deploys will be performed against the k3s cluster.
 
 ### Running a multi-cluster node
 
-> [!IMPORTANT]
-> This is future planned behavior. Commands here will not work as expected.
-
-For high-availability, it is recommended to add both agent and server nodes to the cluster. Dokku will default to starting the cluster with an embedded Etcd database backend, and is ready to add new agent or server nodes immediately.
-
-Multiple server nodes can be added with the `scheduler-k3s:add-server` command. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in server mode.
-
 > [!WARNING]
 > Certain ports must be open for cross-server communication. Refer to the [K3s networking documentation](https://docs.k3s.io/installation/requirements?os=debian#networking) for the required open ports between servers prior to running the command.
 
-```shell
-dokku scheduler-k3s:add-server root@server-1.example.com
-```
-
-Server nodes are typically used to replicate the cluster state, and it is recommended to have an odd number of nodes spread across several availability zones (datacenters in close proximity within a region). This allows for higher availability in the event of a cluster failure. Server nodes do not run any app workloads, but run control-plane services such as the traefik load balancer and the etcd backing store.
-
-> [!NOTE]
-> Only the initial Dokku server will be properly configured for deployment, and should be considered your git remote. Additional server nodes are for ensuring high-availability of the K3s etcd state. Ensure this server is properly backed up and restorable or deployments will not work.
-
-Agent nodes are used to run. To add an agent, run the `add-agent` command. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in agent mode. Agents are typically used to run app workloads.
-
-```shell
-dokku scheduler-k3s:add-agent root@server-1.example.com
-```
+For high-availability, it is recommended to add both agent and server nodes to the cluster. Dokku will default to starting the cluster with an embedded Etcd database backend, and is ready to add new agent or server nodes immediately.
 
 When attaching an agent or server node, the K3s plugin will look at the IP associated with the `eth0` interface and use that to connect the new node to the cluster. To change this, set the `network-interface` property to the appropriate value.
 
@@ -122,54 +102,45 @@ When attaching an agent or server node, the K3s plugin will look at the IP assoc
 dokku scheduler-k3s:set --global network-interface eth1
 ```
 
-Dokku servers may not have an ssh key pair by default, but they can be generated as needed
+Dokku will connect to remote servers via the `root` user with the `dokku` user's SSH key pair. Dokku servers may not have an ssh key pair by default, but they can be generated as needed via the `git:generate-deploy-key` command.
 
 ```shell
 dokku git:generate-deploy-key
 ```
 
-This key can then be displayed with the `git:public-key` command:
+This key can then be displayed with the `git:public-key` command, and copied to the remote server's `/root/.ssh/authorized_keys` file.
 
 ```shell
 dokku git:public-key
 ```
 
-### Joining an existing cluster
-
-> [!IMPORTANT]
-> This is future planned behavior. Commands here will not work as expected.
-
-In some cases, Dokku must be setup from scratch and joined to an existing cluster. To do so, first set the k3s token value:
+Multiple server nodes can be added with the `scheduler-k3s:cluster-add` command. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in server mode.
 
 ```shell
-dokku scheduler-k3s:set --global token topsecret:server:token
-```
-
-Next, run the `scheduler-k3s:cluster-add` command. This command takes a single server node in the cluster, will SSH onto the specified server, retrieve the kubeconfig, and then configure kubectl on the Dokku node to speak with the specified k3s cluster. 
-
-> [!TODO]
-> TODO: Dokku will also periodically retrieve all server nodes in the cluster so that server nodes can be safely replaced as needed.
-
-```shell
-dokku scheduler-k3s:cluster-add ssh://root@worker-1.example.com
-```
-
-If the server isn't in the `known_hosts` file, the connection will fail. This can be bypassed by setting the `--insecure-allow-unknown-hosts` flag:
-
-```shell
-dokku scheduler-k3s:cluster-add --insecure-allow-unknown-hosts ssh://root@worker-1.example.com
-```
-
-New server nodes are added as worker nodes which run application workloads but do not otherwise participate in managing the cluster. For high availability, new server nodes can be added to the cluster by specifying the `--role` flag:
-
-```shell
-dokku scheduler-kes:cluster-add --role server ssh://root@cluster-1.example.com
+dokku scheduler-k3s:cluster-add  --role server ssh://root@server-1.example.com
 ```
 
 Server nodes allow any workloads to be scheduled on them by default, in addition to the control-plane, etcd, and the scheduler itself. To avoid app workloads being scheduled on your control-plane, use the `--taint-scheduling` flag:
 
 ```shell
-dokku scheduler-kes:cluster-add --role server --taint-scheduling ssh://root@cluster-1.example.com
+dokku scheduler-k3s:cluster-add --role server --taint-scheduling ssh://root@cluster-1.example.com
+```
+
+If the server isn't in the `known_hosts` file, the connection will fail. This can be bypassed by setting the `--insecure-allow-unknown-hosts` flag:
+
+```shell
+dokku scheduler-k3s:cluster-add --role server --insecure-allow-unknown-hosts ssh://root@worker-1.example.com
+```
+
+Server nodes are typically used to replicate the cluster state, and it is recommended to have an odd number of nodes spread across several availability zones (datacenters in close proximity within a region). This allows for higher availability in the event of a cluster failure. Server nodes run control-plane services such as the traefik load balancer and the etcd backing store.
+
+> [!NOTE]
+> Only the initial Dokku server will be properly configured for push deployment, and should be considered your git remote. Additional server nodes are for ensuring high-availability of the K3s etcd state. Ensure this server is properly backed up and restorable or deployments will not work.
+
+Agent nodes are used to run. To add an agent, run the `scheduler-k3s:cluster-add` with the `--role worker` flag. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in agent mode. Agents are typically used to run app workloads.
+
+```shell
+dokku scheduler-k3s:cluster-add --role worker ssh://root@worker-1.example.com
 ```
 
 ### Using kubectl remotely

From 34b06d488f4cd67ba543fcb1eb8643e9f3547419 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 05:00:47 -0500
Subject: [PATCH 061/113] fix: add missing makefile and dokku entries for
 uninstall and cluster-add

---
 dokku                          | 4 ++--
 plugins/scheduler-k3s/Makefile | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/dokku b/dokku
index ba7355a05..95b3c2dd5 100755
--- a/dokku
+++ b/dokku
@@ -89,14 +89,14 @@ fi
 
 if [[ $(id -un) != "dokku" ]]; then
   unset TMP TMPDIR TEMP TEMPDIR
-  if [[ ! $1 =~ plugin:* ]] && [[ $1 != "ssh-keys:add" ]] && [[ $1 != "ssh-keys:remove" ]] && [[ $1 != "scheduler-k3s:initialize" ]]; then
+  if [[ ! $1 =~ plugin:* ]] && [[ $1 != "ssh-keys:add" ]] && [[ $1 != "ssh-keys:remove" ]] && [[ $1 != "scheduler-k3s:initialize" ]] && [[ $1 != "scheduler-k3s:uninstall" ]] && [[ $1 != "scheduler-k3s:cluster-add" ]]; then
     export SSH_USER=$(id -un)
     sudo -u dokku -E -H "$0" "$@"
     exit $?
   fi
 fi
 
-if [[ $1 =~ ^plugin:.* && $1 != "plugin:help" && $1 != "plugin:list" ]] || [[ $1 == "ssh-keys:add" ]] || [[ $1 == "ssh-keys:remove" ]] || [[ $1 == "scheduler-k3s:initialize" ]]; then
+if [[ $1 =~ ^plugin:.* && $1 != "plugin:help" && $1 != "plugin:list" ]] || [[ $1 == "ssh-keys:add" ]] || [[ $1 == "ssh-keys:remove" ]] || [[ $1 == "scheduler-k3s:initialize" ]] || [[ $1 == "scheduler-k3s:uninstall" ]] || [[ $1 == "scheduler-k3s:cluster-add" ]]; then
   if [[ $(id -un) != "root" ]]; then
     dokku_log_fail "This command must be run as root"
   else
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index 84c1ed2bd..effd45429 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/initialize subcommands/join-cluster subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
+SUBCOMMANDS = subcommands/initialize subcommands/cluster-add subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
 TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s

From 68fef564033db0abf3e359cc906c4636d84e1b3f Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 05:10:06 -0500
Subject: [PATCH 062/113] docs: remove note about syncing with traefik

---
 docs/deployment/schedulers/k3s.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 0c93c377e..8bafa8f99 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -69,7 +69,7 @@ Additionally, apps should be configured to push images on the release phase via
 dokku registry:set --global push-on-release true
 ```
 
-As routing is handled by traefik managed on the k3s plugin, set the proxy plugin to `k3s` as well. This implements a traefik-plugin compatible layer, allowing users to otherwise interact with the `traefik` plugin as expected.
+As routing is handled by traefik managed on the k3s plugin, set the proxy plugin to `k3s` as well.
 
 ```shell
 dokku proxy:set --global k3s

From 78bc2849cb945cc72e675a548a000155cf2077b4 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 05:49:20 -0500
Subject: [PATCH 063/113] feat: add longhorn

---
 plugins/scheduler-k3s/subcommands.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 6bc94d701..64c045fce 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -211,6 +211,23 @@ func CommandInitialize(taintScheduling bool) error {
 		return fmt.Errorf("Invalid exit code from setfacl command: %d", registryAclCmd.ExitCode)
 	}
 
+	common.LogInfo2Quiet("Installing longhorn")
+	longhornCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: "kubectl",
+		Args: []string{
+			"apply",
+			"-f",
+			"https://raw.githubusercontent.com/longhorn/longhorn/v1.4.0/deploy/longhorn.yaml",
+		},
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call kubectl command: %w", err)
+	}
+	if longhornCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from kubectl command: %d", longhornCmd.ExitCode)
+	}
+
 	common.LogInfo2Quiet("Installing k3s automatic upgrader")
 	upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{
 		Command: "kubectl",

From 707c22eae73c4aaa6a065cac8d70aeeb2639b95b Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 05:49:29 -0500
Subject: [PATCH 064/113] chore: update docs concerning network requests

---
 docs/deployment/schedulers/k3s.md | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 8bafa8f99..0ee1d1fce 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -33,7 +33,15 @@ Initialize the cluster in single-node mode. This will start k3s on the Dokku nod
 dokku scheduler-k3s:initialize
 ```
 
-The above command will initialize a cluster with an automatically generated token for authentication. This token should be stored securely for later recovery, and can be displayed with via the `scheduler-k3s:report` command:
+The above command will initialize a cluster with the following configuration:
+
+- etcd distributed backing store
+- Wireguard as the networking flannel
+- K3s automatic upgrader
+- Longhorn distributed block storage
+- Traefik configured to run on all nodes in the cluster
+
+Additionally, an internal token for authentication will be automatically generated. This token should be stored securely for later recovery, and can be displayed with via the `scheduler-k3s:report` command:
 
 ```shell
 dokku scheduler-k3s:report --global
@@ -89,6 +97,9 @@ dokku scheduler:set --global selected k3s
 
 At this point, all app deploys will be performed against the k3s cluster.
 
+> [!NOTE]
+> HTTP requests for apps can be performed against any node in the cluster. Without extra configuration, many other ports may also be available on the host. For security reasons, it may be desirable to place the k3s cluster behind one or more TCP load balancers while shutting off traffic to all cluster ports. Please consult your hosting provider for more information on how to provision a TCP load balancer and shut off all ports other than 22/80/443 access to the outside world.
+
 ### Running a multi-cluster node
 
 > [!WARNING]

From 09ca1f1299678a036ab70aab6ef0f9b394eab676 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 06:56:45 -0500
Subject: [PATCH 065/113] fix: correct issue with file copygin

---
 plugins/common/sftp.go | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/plugins/common/sftp.go b/plugins/common/sftp.go
index d1e692f1e..586037da6 100644
--- a/plugins/common/sftp.go
+++ b/plugins/common/sftp.go
@@ -18,20 +18,17 @@ import (
 
 // SftpCopyInput is the input for CallSftpCopy
 type SftpCopyInput struct {
-	// SourcePath is the path to the file to copy
-	SourcePath string
+	// AllowUknownHosts allows connecting to hosts with unknown host keys
+	AllowUknownHosts bool
 
 	// DestinationPath is the path to copy the file to
 	DestinationPath string
 
-	// AllowUknownHosts allows connecting to hosts with unknown host keys
-	AllowUknownHosts bool
-
 	// RemoteHost is the remote host to connect to
 	RemoteHost string
 
-	// Sudo runs the command with sudo -n -u root
-	Sudo bool
+	// SourcePath is the path to the file to copy
+	SourcePath string
 }
 
 // CallSftpCopy copies a file to a remote host via sftp
@@ -226,14 +223,7 @@ func (task SftpCopyTask) Execute(ctx context.Context) (SftpCopyResult, error) {
 		return SftpCopyResult{}, fmt.Errorf("failed to create sftp client: %w", err)
 	}
 
-	srcFile, err := os.Open(task.SourcePath)
-	if err != nil {
-		return SftpCopyResult{}, fmt.Errorf("failed to open source file: %w", err)
-	}
-	defer srcFile.Close()
-
-	var contents []byte
-	_, err = srcFile.Read(contents)
+	contents, err := os.ReadFile(task.SourcePath)
 	if err != nil {
 		return SftpCopyResult{}, fmt.Errorf("failed to read source file: %w", err)
 	}

From a354210f67706ab59fd3997b92ba3c4ecce7ab13 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 06:57:01 -0500
Subject: [PATCH 066/113] fix: treat the registry file as a pointer

---
 plugins/scheduler-k3s/triggers.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 5cd66fd70..0b2e9d3e2 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -86,13 +86,13 @@ func TriggerPostRegistryLogin(server string, username string) error {
 
 	password := os.Getenv("DOCKER_REGISTRY_PASS")
 
-	registry := registries.Registry{}
 	yamlFile, err := os.ReadFile(RegistryConfigPath)
 	if err != nil {
 		return fmt.Errorf("Unable to read existing registries.yaml: %w", err)
 	}
 
-	err = yaml.Unmarshal(yamlFile, registry)
+	var registry registries.Registry
+	err = yaml.Unmarshal(yamlFile, &registry)
 	if err != nil {
 		return fmt.Errorf("Unable to unmarshal registry configuration from yaml: %w", err)
 	}

From 3375474a4ce077f4f427319ae194dc6b8c8ce5d7 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sat, 20 Jan 2024 06:57:20 -0500
Subject: [PATCH 067/113] feat: copy the registry file to all remote hosts

---
 plugins/scheduler-k3s/go.mod         |  4 +-
 plugins/scheduler-k3s/k8s.go         | 37 +++++++++++--
 plugins/scheduler-k3s/registry.go    | 78 ++++++++++++++++++++++++++++
 plugins/scheduler-k3s/subcommands.go | 54 ++++++++++++-------
 plugins/scheduler-k3s/triggers.go    | 29 ++++++++++-
 5 files changed, 176 insertions(+), 26 deletions(-)
 create mode 100644 plugins/scheduler-k3s/registry.go

diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index edd7329dc..5ce122cd9 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -8,7 +8,9 @@ require (
 	github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
 	github.com/fatih/color v1.16.0
+	github.com/go-openapi/jsonpointer v0.19.6
 	github.com/go-resty/resty/v2 v2.11.0
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/rancher/wharfie v0.6.5
 	github.com/ryanuber/columnize v2.1.2+incompatible
@@ -66,7 +68,6 @@ require (
 	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
@@ -84,7 +85,6 @@ require (
 	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go
index 49accdbeb..2ccfee834 100644
--- a/plugins/scheduler-k3s/k8s.go
+++ b/plugins/scheduler-k3s/k8s.go
@@ -73,6 +73,37 @@ func KubernetesClientConfig() clientcmd.ClientConfig {
 		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
 }
 
+// AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
+type AnnotateNodeInput struct {
+	// Name is the Kubernetes node name
+	Name string
+	// Key is the annotation key
+	Key string
+	// Value is the annotation value
+	Value string
+}
+
+// AnnotateNode annotates a Kubernetes node
+func (k KubernetesClient) AnnotateNode(ctx context.Context, input AnnotateNodeInput) error {
+	node, err := k.Client.CoreV1().Nodes().Get(ctx, input.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if node == nil {
+		return errors.New("node is nil")
+	}
+
+	keyPath := fmt.Sprintf("/metadata/annotations/%s", jsonpointer.Escape(input.Key))
+	patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value)
+	_, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to annotate node: %w", err)
+	}
+
+	return nil
+}
+
 // CreateJobInput contains all the information needed to create a Kubernetes job
 type CreateJobInput struct {
 	// Job is the Kubernetes job
@@ -190,9 +221,9 @@ func (k KubernetesClient) LabelNode(ctx context.Context, input LabelNodeInput) e
 		return errors.New("node is nil")
 	}
 
-	keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape("kubernetes.io/role"))
-	patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, "worker")
-	_, err = k.Client.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
+	keyPath := fmt.Sprintf("/metadata/labels/%s", jsonpointer.Escape(input.Key))
+	patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, input.Value)
+	_, err = k.Client.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
 	if err != nil {
 		return fmt.Errorf("failed to label node: %w", err)
 	}
diff --git a/plugins/scheduler-k3s/registry.go b/plugins/scheduler-k3s/registry.go
new file mode 100644
index 000000000..824e1b7b4
--- /dev/null
+++ b/plugins/scheduler-k3s/registry.go
@@ -0,0 +1,78 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+func copyRegistryToNode(ctx context.Context, remoteHost string) error {
+	common.LogInfo1Quiet(fmt.Sprintf("Updating k3s registry configuration on %s", remoteHost))
+	sftpCmd, err := common.CallSftpCopy(common.SftpCopyInput{
+		AllowUknownHosts: true,
+		DestinationPath:  "/tmp/registries.yaml",
+		RemoteHost:       remoteHost,
+		SourcePath:       RegistryConfigPath,
+	})
+	if err != nil {
+		return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
+	}
+
+	if sftpCmd.ExitErr != nil {
+		return fmt.Errorf("Error copying registries.yaml to remote host: %s", sftpCmd.ExitErr.Error())
+	}
+
+	common.LogVerboseQuiet("Moving k3s registry configuration into place")
+	mvCmd, err := common.CallSshCommand(common.SshCommandInput{
+		AllowUknownHosts: true,
+		Args:             []string{"/tmp/registries.yaml", RegistryConfigPath},
+		Command:          "mv",
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+		Sudo:             true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call mv command over ssh: %w", err)
+	}
+
+	if mvCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from mv command over ssh: %d", mvCmd.ExitCode)
+	}
+
+	common.LogVerboseQuiet("Updating k3s registry permissions")
+	chmodCmd, err := common.CallSshCommand(common.SshCommandInput{
+		AllowUknownHosts: true,
+		Args:             []string{"0644", RegistryConfigPath},
+		Command:          "chmod",
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+		Sudo:             true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call chmod command over ssh: %w", err)
+	}
+
+	if chmodCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode)
+	}
+
+	common.LogVerboseQuiet("Updating k3s registry ower")
+	chownCmd, err := common.CallSshCommand(common.SshCommandInput{
+		AllowUknownHosts: true,
+		Args:             []string{"root:root", RegistryConfigPath},
+		Command:          "chown",
+		RemoteHost:       remoteHost,
+		StreamStdio:      true,
+		Sudo:             true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call chown command over ssh: %w", err)
+	}
+
+	if chownCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from chown command over ssh: %d", chownCmd.ExitCode)
+	}
+
+	return nil
+}
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 64c045fce..31060faf8 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -462,7 +462,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
 	}
 
-	common.LogInfo2Quiet("Joining k3s cluster")
+	common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
 	joinCmd, err := common.CallSshCommand(common.SshCommandInput{
 		Command:          "/tmp/k3s-installer.sh",
 		Args:             args,
@@ -477,27 +477,31 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 	if joinCmd.ExitCode != 0 {
 		return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
 	}
+	ctx := context.Background()
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	common.LogInfo2Quiet("Waiting for node to exist")
+	nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
+		Clientset:  clientset,
+		NodeName:   nodeName,
+		RetryCount: 20,
+	})
+	if err != nil {
+		return fmt.Errorf("Error waiting for pod to exist: %w", err)
+	}
+	if len(nodes) == 0 {
+		return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled appropriately and cannot access registry secrets")
+	}
+
+	err = copyRegistryToNode(ctx, remoteHost)
+	if err != nil {
+		return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
+	}
 
 	if role == "worker" {
-		ctx := context.Background()
-		clientset, err := NewKubernetesClient()
-		if err != nil {
-			return fmt.Errorf("Unable to create kubernetes client: %w", err)
-		}
-
-		common.LogInfo2Quiet("Waiting for node to exist")
-		nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
-			Clientset:  clientset,
-			NodeName:   nodeName,
-			RetryCount: 20,
-		})
-		if err != nil {
-			return fmt.Errorf("Error waiting for pod to exist: %w", err)
-		}
-		if len(nodes) == 0 {
-			return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled kubernetes.io/role=worker")
-		}
-
 		common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker")
 		err = clientset.LabelNode(ctx, LabelNodeInput{
 			Name:  nodes[0].Name,
@@ -509,6 +513,16 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		}
 	}
 
+	common.LogInfo2Quiet("Annotating node with connection information")
+	err = clientset.AnnotateNode(ctx, AnnotateNodeInput{
+		Name:  nodes[0].Name,
+		Key:   "dokku.com/remote-host",
+		Value: remoteHost,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to patch node: %w", err)
+	}
+
 	common.LogVerboseQuiet("Done")
 	return nil
 }
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 0b2e9d3e2..4c0368c50 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -21,6 +21,7 @@ import (
 	"github.com/dokku/dokku/plugins/config"
 	"github.com/dokku/dokku/plugins/cron"
 	"github.com/fatih/color"
+	"github.com/hashicorp/go-multierror"
 	"github.com/kballard/go-shellquote"
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/ryanuber/columnize"
@@ -121,7 +122,33 @@ func TriggerPostRegistryLogin(server string, username string) error {
 	}
 
 	// todo: auth against all nodes in cluster
-	return nil
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	ctx := context.Background()
+	nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
+	if err != nil {
+		return fmt.Errorf("Error listing nodes: %w", err)
+	}
+
+	var result error
+	for _, node := range nodes {
+		remoteHost, ok := node.Annotations["dokku.com/remote-host"]
+		if !ok {
+			continue
+		}
+
+		err := copyRegistryToNode(ctx, remoteHost)
+		if err != nil {
+			wrappedErr := fmt.Errorf("Error copying registry to node: %w", err)
+			result = multierror.Append(result, wrappedErr)
+			common.LogWarn(wrappedErr.Error())
+		}
+	}
+
+	return result
 }
 
 // TriggerSchedulerDeploy deploys an image tag for a given application

From 33a3e6a2fcc5787ce468a13adbd0ac969f2d3639 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sun, 21 Jan 2024 23:05:07 -0500
Subject: [PATCH 068/113] chore: drop unused node label

---
 plugins/scheduler-k3s/subcommands.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 31060faf8..164f9db35 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -418,8 +418,6 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 	args := []string{
 		// disable local-storage
 		"--disable", "local-storage",
-		// add a node label
-		"--node-label=node_type=worker",
 		// use wireguard for flannel
 		"--flannel-backend=wireguard-native",
 		// specify the node name

From d494e01c7ba29f6611df30dd0fb28f59cfd59297 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sun, 21 Jan 2024 23:05:20 -0500
Subject: [PATCH 069/113] chore: update error message

---
 plugins/scheduler-k3s/subcommands.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 164f9db35..5a398bf1e 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -491,7 +491,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		return fmt.Errorf("Error waiting for pod to exist: %w", err)
 	}
 	if len(nodes) == 0 {
-		return fmt.Errorf("Unable to find node after joining cluster, node will not be labeled appropriately and cannot access registry secrets")
+		return fmt.Errorf("Unable to find node after joining cluster, node will not be annotated/labeled appropriately access registry secrets")
 	}
 
 	err = copyRegistryToNode(ctx, remoteHost)

From 0730647af72f2ef37e7035aa986c03439258367b Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Sun, 21 Jan 2024 23:48:38 -0500
Subject: [PATCH 070/113] feat: implement :cluster-list and :cluster-remove

These - and the existing :cluster-add command - do not require root.
---
 dokku                                         |   4 +-
 plugins/scheduler-k3s/Makefile                |   2 +-
 plugins/scheduler-k3s/functions.go            |  84 ++++++++++++++-
 plugins/scheduler-k3s/k8s.go                  |  35 ++++++
 .../src/subcommands/subcommands.go            |  10 ++
 plugins/scheduler-k3s/subcommands.go          | 102 ++++++++++++++++++
 6 files changed, 229 insertions(+), 8 deletions(-)

diff --git a/dokku b/dokku
index 95b3c2dd5..46f8d6f92 100755
--- a/dokku
+++ b/dokku
@@ -89,14 +89,14 @@ fi
 
 if [[ $(id -un) != "dokku" ]]; then
   unset TMP TMPDIR TEMP TEMPDIR
-  if [[ ! $1 =~ plugin:* ]] && [[ $1 != "ssh-keys:add" ]] && [[ $1 != "ssh-keys:remove" ]] && [[ $1 != "scheduler-k3s:initialize" ]] && [[ $1 != "scheduler-k3s:uninstall" ]] && [[ $1 != "scheduler-k3s:cluster-add" ]]; then
+  if [[ ! $1 =~ plugin:* ]] && [[ $1 != "ssh-keys:add" ]] && [[ $1 != "ssh-keys:remove" ]] && [[ $1 != "scheduler-k3s:initialize" ]] && [[ $1 != "scheduler-k3s:uninstall" ]]; then
     export SSH_USER=$(id -un)
     sudo -u dokku -E -H "$0" "$@"
     exit $?
   fi
 fi
 
-if [[ $1 =~ ^plugin:.* && $1 != "plugin:help" && $1 != "plugin:list" ]] || [[ $1 == "ssh-keys:add" ]] || [[ $1 == "ssh-keys:remove" ]] || [[ $1 == "scheduler-k3s:initialize" ]] || [[ $1 == "scheduler-k3s:uninstall" ]] || [[ $1 == "scheduler-k3s:cluster-add" ]]; then
+if [[ $1 =~ ^plugin:.* && $1 != "plugin:help" && $1 != "plugin:list" ]] || [[ $1 == "ssh-keys:add" ]] || [[ $1 == "ssh-keys:remove" ]] || [[ $1 == "scheduler-k3s:initialize" ]] || [[ $1 == "scheduler-k3s:uninstall" ]]; then
   if [[ $(id -un) != "root" ]]; then
     dokku_log_fail "This command must be run as root"
   else
diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile
index effd45429..2a41b159a 100644
--- a/plugins/scheduler-k3s/Makefile
+++ b/plugins/scheduler-k3s/Makefile
@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/initialize subcommands/cluster-add subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
+SUBCOMMANDS = subcommands/initialize subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
 TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s
diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 2ebf51e7b..c521f68c7 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -20,13 +22,48 @@ import (
 	"mvdan.cc/sh/v3/shell"
 )
 
+// EnterPodInput contains all the information needed to enter a pod
 type EnterPodInput struct {
-	Clientset             KubernetesClient
-	Command               []string
-	Entrypoint            string
+	// Clientset is the kubernetes clientset
+	Clientset KubernetesClient
+
+	// Command is the command to run
+	Command []string
+
+	// Entrypoint is the entrypoint to run
+	Entrypoint string
+
+	// SelectedContainerName is the container name to enter
 	SelectedContainerName string
-	SelectedPod           v1.Pod
-	WaitTimeout           int
+
+	// SelectedPod is the pod to enter
+	SelectedPod v1.Pod
+
+	// WaitTimeout is the timeout to wait for the pod to be ready
+	WaitTimeout int
+}
+
+// Node contains information about a node
+type Node struct {
+	// Name is the name of the node
+	Name string
+
+	// Roles is the roles of the node
+	Roles []string
+
+	// Ready is whether the node is ready
+	Ready bool
+
+	// RemoteHost is the remote host
+	RemoteHost string
+
+	// Version is the version of the node
+	Version string
+}
+
+// String returns a string representation of the node
+func (n Node) String() string {
+	return fmt.Sprintf("%s|%s|%s|%s", n.Name, strconv.FormatBool(n.Ready), strings.Join(n.Roles, ","), n.Version)
 }
 
 // StartCommandInput contains all the information needed to get the start command
@@ -323,6 +360,43 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp
 	}
 }
 
+// kubernetesNodeToNode converts a kubernetes node to a Node
+func kubernetesNodeToNode(node v1.Node) Node {
+	roles := []string{}
+	if len(node.Labels["kubernetes.io/role"]) > 0 {
+		roles = append(roles, node.Labels["kubernetes.io/role"])
+	} else {
+		for k, v := range node.Labels {
+			if strings.HasPrefix(k, "node-role.kubernetes.io/") && v == "true" {
+				roles = append(roles, strings.TrimPrefix(k, "node-role.kubernetes.io/"))
+			}
+		}
+	}
+
+	sort.Strings(roles)
+
+	ready := false
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == "Ready" {
+			ready = condition.Status == "True"
+			break
+		}
+	}
+
+	remoteHost := ""
+	if val, ok := node.Annotations["dokku.com/remote-host"]; ok {
+		remoteHost = val
+	}
+
+	return Node{
+		Name:       node.Name,
+		Roles:      roles,
+		Ready:      ready,
+		RemoteHost: remoteHost,
+		Version:    node.Status.NodeInfo.KubeletVersion,
+	}
+}
+
 func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
 	pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
 		Clientset:     input.Clientset,
diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go
index 2ccfee834..fa05e8fc7 100644
--- a/plugins/scheduler-k3s/k8s.go
+++ b/plugins/scheduler-k3s/k8s.go
@@ -163,6 +163,17 @@ func (k KubernetesClient) DeleteJob(ctx context.Context, input DeleteJobInput) e
 	})
 }
 
+// DeleteNodeInput contains all the information needed to delete a Kubernetes node
+type DeleteNodeInput struct {
+	// Name is the Kubernetes node name
+	Name string
+}
+
+// DeleteNode deletes a Kubernetes node
+func (k KubernetesClient) DeleteNode(ctx context.Context, input DeleteNodeInput) error {
+	return k.Client.CoreV1().Nodes().Delete(ctx, input.Name, metav1.DeleteOptions{})
+}
+
 // DeleteSecretInput contains all the information needed to delete a Kubernetes secret
 type DeleteSecretInput struct {
 	// Name is the Kubernetes secret name
@@ -177,6 +188,30 @@ func (k KubernetesClient) DeleteSecret(ctx context.Context, input DeleteSecretIn
 	return k.Client.CoreV1().Secrets(input.Namespace).Delete(ctx, input.Name, metav1.DeleteOptions{})
 }
 
+// GetNodeInput contains all the information needed to get a Kubernetes node
+type GetNodeInput struct {
+	// Name is the Kubernetes node name
+	Name string
+}
+
+// GetNode gets a Kubernetes node
+func (k KubernetesClient) GetNode(ctx context.Context, input GetNodeInput) (Node, error) {
+	if input.Name == "" {
+		return Node{}, errors.New("node name is required")
+	}
+
+	node, err := k.Client.CoreV1().Nodes().Get(ctx, input.Name, metav1.GetOptions{})
+	if err != nil {
+		return Node{}, err
+	}
+
+	if node == nil {
+		return Node{}, errors.New("node is nil")
+	}
+
+	return kubernetesNodeToNode(*node), err
+}
+
 // GetJobInput contains all the information needed to get a Kubernetes job
 type GetPodInput struct {
 	// Name is the Kubernetes pod name
diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go
index 04a9f7dc0..60bf79e23 100644
--- a/plugins/scheduler-k3s/src/subcommands/subcommands.go
+++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go
@@ -31,6 +31,16 @@ func main() {
 		args.Parse(os.Args[2:])
 		remoteHost := args.Arg(0)
 		err = scheduler_k3s.CommandClusterAdd(*role, remoteHost, *allowUknownHosts, *taintScheduling)
+	case "cluster-list":
+		args := flag.NewFlagSet("scheduler-k3s:cluster-list", flag.ExitOnError)
+		format := args.String("format", "stdout", "format: [ stdout | json ]")
+		args.Parse(os.Args[2:])
+		err = scheduler_k3s.CommandClusterList(*format)
+	case "cluster-remove":
+		args := flag.NewFlagSet("scheduler-k3s:cluster-remove", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		nodeName := args.Arg(0)
+		err = scheduler_k3s.CommandClusterRemove(nodeName)
 	case "report":
 		args := flag.NewFlagSet("scheduler-k3s:report", flag.ExitOnError)
 		format := args.String("format", "stdout", "format: [ stdout | json ]")
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 5a398bf1e..522224d0d 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -3,6 +3,7 @@ package scheduler_k3s
 import (
 	"context"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"net"
 	"net/url"
@@ -11,6 +12,7 @@ import (
 
 	"github.com/dokku/dokku/plugins/common"
 	resty "github.com/go-resty/resty/v2"
+	"github.com/ryanuber/columnize"
 )
 
 // CommandInitialize initializes a k3s cluster on the local server
@@ -525,6 +527,106 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 	return nil
 }
 
+// CommandClusterList lists the nodes in the k3s cluster
+func CommandClusterList(format string) error {
+	if format != "stdout" && format != "json" {
+		return fmt.Errorf("Invalid format: %s", format)
+	}
+	if err := isK3sInstalled(); err != nil {
+		return fmt.Errorf("k3s not installed, cannot list cluster nodes")
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	nodes, err := clientset.ListNodes(context.Background(), ListNodesInput{})
+	if err != nil {
+		return fmt.Errorf("Unable to list nodes: %w", err)
+	}
+
+	output := []Node{}
+	for _, node := range nodes {
+		output = append(output, kubernetesNodeToNode(node))
+	}
+
+	if format == "stdout" {
+		lines := []string{"name|ready|roles|version"}
+		for _, node := range output {
+			lines = append(lines, node.String())
+		}
+
+		columnized := columnize.SimpleFormat(lines)
+		fmt.Println(columnized)
+		return nil
+	}
+
+	b, err := json.Marshal(output)
+	if err != nil {
+		return fmt.Errorf("Unable to marshal json: %w", err)
+	}
+
+	fmt.Println(string(b))
+	return nil
+}
+
+// CommandClusterRemove removes a node from the k3s cluster
+func CommandClusterRemove(nodeName string) error {
+	if err := isK3sInstalled(); err != nil {
+		return fmt.Errorf("k3s not installed, cannot remove node")
+	}
+
+	common.LogInfo1Quiet(fmt.Sprintf("Removing %s from k3s cluster", nodeName))
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	common.LogVerboseQuiet("Getting node remote connection information")
+	ctx := context.Background()
+	node, err := clientset.GetNode(ctx, GetNodeInput{
+		Name: nodeName,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to get node: %w", err)
+	}
+
+	common.LogVerboseQuiet("Checking if node is a remote node managed by Dokku")
+	if node.RemoteHost == "" {
+		return fmt.Errorf("Node %s is not a remote node managed by Dokku", nodeName)
+	}
+
+	common.LogVerboseQuiet("Uninstalling k3s on remote host")
+	removeCmd, err := common.CallSshCommand(common.SshCommandInput{
+		Command:          "/usr/local/bin/k3s-uninstall.sh",
+		Args:             []string{},
+		AllowUknownHosts: true,
+		RemoteHost:       node.RemoteHost,
+		StreamStdio:      true,
+		Sudo:             true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call k3s uninstall command over ssh: %w", err)
+	}
+
+	if removeCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from k3s uninstall command over ssh: %d", removeCmd.ExitCode)
+	}
+
+	common.LogVerboseQuiet("Deleting node from k3s cluster")
+	err = clientset.DeleteNode(ctx, DeleteNodeInput{
+		Name: nodeName,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to delete node: %w", err)
+	}
+
+	common.LogVerboseQuiet("Done")
+
+	return nil
+}
+
 // CommandReport displays a scheduler-k3s report for one or more apps
 func CommandReport(appName string, format string, infoFlag string) error {
 	if len(appName) == 0 {

From 309256955de8acaab2f7e4c4afdf8ab53f493286 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 00:41:51 -0500
Subject: [PATCH 071/113] feat: label the k3s server nodes as capable of
 fronting traffic

The idea here is that only k3s nodes can/should handle certificate requests. While users may wish to force these onto dedicated lb nodes in the future, at least for now, separating app workloads with routing seems like an okay trade-off.
---
 plugins/scheduler-k3s/subcommands.go | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 522224d0d..321505aba 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -246,6 +246,20 @@ func CommandInitialize(taintScheduling bool) error {
 	if upgradeCmd.ExitCode != 0 {
 		return fmt.Errorf("Invalid exit code from kubectl command: %d", upgradeCmd.ExitCode)
 	}
+	common.LogInfo2Quiet("Labeling node svccontroller.k3s.cattle.io/enablelb=true")
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	err = clientset.LabelNode(context.Background(), LabelNodeInput{
+		Name:  nodeName,
+		Key:   "svccontroller.k3s.cattle.io/enablelb",
+		Value: "true",
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to patch node: %w", err)
+	}
 
 	common.LogVerboseQuiet("Done")
 
@@ -511,6 +525,16 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		if err != nil {
 			return fmt.Errorf("Unable to patch node: %w", err)
 		}
+	} else {
+		common.LogInfo2Quiet("Labeling node svccontroller.k3s.cattle.io/enablelb=true")
+		err = clientset.LabelNode(ctx, LabelNodeInput{
+			Name:  nodes[0].Name,
+			Key:   "svccontroller.k3s.cattle.io/enablelb",
+			Value: "true",
+		})
+		if err != nil {
+			return fmt.Errorf("Unable to patch node: %w", err)
+		}
 	}
 
 	common.LogInfo2Quiet("Annotating node with connection information")

From e4a74366badfcc201b945c519897a17787f544cc Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 00:45:40 -0500
Subject: [PATCH 072/113] chore: move around structs to be closer to the
 functions using them

---
 plugins/scheduler-k3s/template.go | 166 +++++++++++++++---------------
 1 file changed, 83 insertions(+), 83 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 1b9de2abc..e1a7e36bb 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -31,86 +31,6 @@ type Chart struct {
 	Version    string `yaml:"version"`
 }
 
-type CreateIngressRoutesInput struct {
-	AppName    string
-	ChartDir   string
-	Deployment appsv1.Deployment
-	Namespace  string
-	PortMaps   []PortMap
-	Service    v1.Service
-}
-
-type Deployment struct {
-	AppName          string
-	Command          []string
-	Image            string
-	ImagePullSecrets string
-	ImageSourceType  string
-	Healthchecks     []appjson.Healthcheck
-	Namespace        string
-	PrimaryPort      int32
-	PortMaps         []PortMap
-	ProcessType      string
-	Replicas         int32
-	WorkingDir       string
-}
-
-type IngressRouteEntrypoint string
-
-const (
-	IngressRouteEntrypoint_HTTP  IngressRouteEntrypoint = "web"
-	IngressRouteEntrypoint_HTTPS IngressRouteEntrypoint = "websecure"
-)
-
-type IngressRoute struct {
-	AppName     string
-	Entrypoints []IngressRouteEntrypoint
-	Hostnames   []string
-	Namespace   string
-	PortMap     PortMap
-	ProcessType string
-	ServiceName string
-}
-
-type Job struct {
-	AppName          string
-	Command          []string
-	DeploymentID     int64
-	Entrypoint       string
-	Env              map[string]string
-	ID               string
-	Image            string
-	ImagePullSecrets string
-	ImageSourceType  string
-	Interactive      bool
-	Labels           map[string]string
-	Namespace        string
-	ProcessType      string
-	Schedule         string
-	Suffix           string
-	RemoveContainer  bool
-	WorkingDir       string
-}
-
-type Secret struct {
-	AppName   string
-	Env       map[string]string
-	Namespace string
-}
-
-type Service struct {
-	AppName   string
-	Namespace string
-	PortMaps  []PortMap
-}
-
-type WriteResourceInput struct {
-	AppendContents string
-	Object         runtime.Object
-	Path           string
-	Replacements   *orderedmap.OrderedMap[string, string]
-}
-
 type Values struct {
 	DeploymentID string                   `yaml:"deploment_id"`
 	Secrets      map[string]string        `yaml:"secrets"`
@@ -121,9 +41,13 @@ type ValuesProcess struct {
 	Replicas int32 `yaml:"replicas"`
 }
 
-type WriteYamlInput struct {
-	Object interface{}
-	Path   string
+type CreateIngressRoutesInput struct {
+	AppName    string
+	ChartDir   string
+	Deployment appsv1.Deployment
+	Namespace  string
+	PortMaps   []PortMap
+	Service    v1.Service
 }
 
 func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
@@ -169,6 +93,26 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 	return nil
 }
 
+type Job struct {
+	AppName          string
+	Command          []string
+	DeploymentID     int64
+	Entrypoint       string
+	Env              map[string]string
+	ID               string
+	Image            string
+	ImagePullSecrets string
+	ImageSourceType  string
+	Interactive      bool
+	Labels           map[string]string
+	Namespace        string
+	ProcessType      string
+	Schedule         string
+	Suffix           string
+	RemoveContainer  bool
+	WorkingDir       string
+}
+
 func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 	if input.Schedule == "" {
 		return batchv1.CronJob{}, fmt.Errorf("Schedule cannot be empty")
@@ -343,6 +287,21 @@ func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
 	return job, nil
 }
 
+type Deployment struct {
+	AppName          string
+	Command          []string
+	Image            string
+	ImagePullSecrets string
+	ImageSourceType  string
+	Healthchecks     []appjson.Healthcheck
+	Namespace        string
+	PrimaryPort      int32
+	PortMaps         []PortMap
+	ProcessType      string
+	Replicas         int32
+	WorkingDir       string
+}
+
 func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	labels := map[string]string{
 		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
@@ -565,6 +524,23 @@ func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 	return deployment, nil
 }
 
+type IngressRouteEntrypoint string
+
+const (
+	IngressRouteEntrypoint_HTTP  IngressRouteEntrypoint = "web"
+	IngressRouteEntrypoint_HTTPS IngressRouteEntrypoint = "websecure"
+)
+
+type IngressRoute struct {
+	AppName     string
+	Entrypoints []IngressRouteEntrypoint
+	Hostnames   []string
+	Namespace   string
+	PortMap     PortMap
+	ProcessType string
+	ServiceName string
+}
+
 func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressRoute {
 	entryPoint := IngressRouteEntrypoint_HTTP
 	if input.PortMap.Scheme == "https" {
@@ -768,6 +744,12 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 	return job, nil
 }
 
+type Secret struct {
+	AppName   string
+	Env       map[string]string
+	Namespace string
+}
+
 func templateKubernetesSecret(input Secret) corev1.Secret {
 	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
 	labels := map[string]string{
@@ -793,6 +775,12 @@ func templateKubernetesSecret(input Secret) corev1.Secret {
 	return secret
 }
 
+type Service struct {
+	AppName   string
+	Namespace string
+	PortMaps  []PortMap
+}
+
 func templateKubernetesService(input Service) corev1.Service {
 	labels := map[string]string{
 		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, "web"),
@@ -830,6 +818,13 @@ func templateKubernetesService(input Service) corev1.Service {
 	return service
 }
 
+type WriteResourceInput struct {
+	AppendContents string
+	Object         runtime.Object
+	Path           string
+	Replacements   *orderedmap.OrderedMap[string, string]
+}
+
 func writeResourceToFile(input WriteResourceInput) error {
 	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
 	printr := printers.NewTypeSetter(runtimeScheme).ToPrinter(&printers.YAMLPrinter{})
@@ -880,6 +875,11 @@ func writeResourceToFile(input WriteResourceInput) error {
 	return nil
 }
 
+type WriteYamlInput struct {
+	Object interface{}
+	Path   string
+}
+
 func writeYaml(input WriteYamlInput) error {
 	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
 	data, err := yaml.Marshal(input.Object)

From f7acbb1e7c4d0d137e72b473a15d1f6eb5a0ef91 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 01:25:01 -0500
Subject: [PATCH 073/113] refactor: place all required manifests in a single
 location

This will make it easier for us to apply the installs.
---
 plugins/scheduler-k3s/k8s.go           | 25 ++++++++++++++++
 plugins/scheduler-k3s/scheduler_k3s.go | 19 ++++++++++++
 plugins/scheduler-k3s/subcommands.go   | 41 ++++++--------------------
 3 files changed, 53 insertions(+), 32 deletions(-)

diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go
index fa05e8fc7..413309f95 100644
--- a/plugins/scheduler-k3s/k8s.go
+++ b/plugins/scheduler-k3s/k8s.go
@@ -104,6 +104,31 @@ func (k KubernetesClient) AnnotateNode(ctx context.Context, input AnnotateNodeIn
 	return nil
 }
 
+type ApplyKubernetesManifestInput struct {
+	// Manifest is the path to the Kubernetes manifest
+	Manifest string
+}
+
+func (k KubernetesClient) ApplyKubernetesManifest(ctx context.Context, input ApplyKubernetesManifestInput) error {
+	upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{
+		Command: "kubectl",
+		Args: []string{
+			"apply",
+			"-f",
+			input.Manifest,
+		},
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Unable to call kubectl command: %w", err)
+	}
+	if upgradeCmd.ExitCode != 0 {
+		return fmt.Errorf("Invalid exit code from kubectl command: %d", upgradeCmd.ExitCode)
+	}
+
+	return nil
+}
+
 // CreateJobInput contains all the information needed to create a Kubernetes job
 type CreateJobInput struct {
 	// Job is the Kubernetes job
diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 715d37fac..37228e64e 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -43,6 +43,25 @@ var (
 
 var k8sNativeSchemeOnce sync.Once
 
+type Manifest struct {
+	Name    string
+	Version string
+	Path    string
+}
+
+var Manifests = []Manifest{
+	{
+		Name:    "longhorn",
+		Version: "1.5.3",
+		Path:    "https://raw.githubusercontent.com/longhorn/longhorn/v1.5.3/deploy/longhorn.yaml",
+	},
+	{
+		Name:    "system-upgrader",
+		Version: "0.13.2",
+		Path:    "https://github.com/rancher/system-upgrade-controller/releases/v0.13.2/download/system-upgrade-controller.yaml",
+	},
+}
+
 func init() {
 	k8sNativeSchemeOnce.Do(func() {
 		_ = appsv1.AddToScheme(runtimeScheme)
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 321505aba..1e94e4311 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -213,41 +213,18 @@ func CommandInitialize(taintScheduling bool) error {
 		return fmt.Errorf("Invalid exit code from setfacl command: %d", registryAclCmd.ExitCode)
 	}
 
-	common.LogInfo2Quiet("Installing longhorn")
-	longhornCmd, err := common.CallExecCommand(common.ExecCommandInput{
-		Command: "kubectl",
-		Args: []string{
-			"apply",
-			"-f",
-			"https://raw.githubusercontent.com/longhorn/longhorn/v1.4.0/deploy/longhorn.yaml",
-		},
-		StreamStdio: true,
-	})
-	if err != nil {
-		return fmt.Errorf("Unable to call kubectl command: %w", err)
-	}
-	if longhornCmd.ExitCode != 0 {
-		return fmt.Errorf("Invalid exit code from kubectl command: %d", longhornCmd.ExitCode)
+	clientset, err := NewKubernetesClient()
+	for _, manifest := range Manifests {
+		common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version))
+		err = clientset.ApplyKubernetesManifest(context.Background(), ApplyKubernetesManifestInput{
+			Manifest: manifest.Path,
+		})
+		if err != nil {
+			return fmt.Errorf("Unable to apply kubernetes manifest: %w", err)
+		}
 	}
 
-	common.LogInfo2Quiet("Installing k3s automatic upgrader")
-	upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{
-		Command: "kubectl",
-		Args: []string{
-			"apply",
-			"-f",
-			"https://github.com/rancher/system-upgrade-controller/releases/latest/download/system-upgrade-controller.yaml",
-		},
-		StreamStdio: true,
-	})
-	if err != nil {
-		return fmt.Errorf("Unable to call kubectl command: %w", err)
-	}
-	if upgradeCmd.ExitCode != 0 {
-		return fmt.Errorf("Invalid exit code from kubectl command: %d", upgradeCmd.ExitCode)
-	}
 	common.LogInfo2Quiet("Labeling node svccontroller.k3s.cattle.io/enablelb=true")
-	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
 	}

From 3ddfd2bc7d0558b5cce6ac4af47a0dd549fc5f32 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 01:30:41 -0500
Subject: [PATCH 074/113] feat: install cert-manager

---
 plugins/scheduler-k3s/scheduler_k3s.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 37228e64e..af971b957 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -60,6 +60,11 @@ var Manifests = []Manifest{
 		Version: "0.13.2",
 		Path:    "https://github.com/rancher/system-upgrade-controller/releases/v0.13.2/download/system-upgrade-controller.yaml",
 	},
+	{
+		Name:    "cert-manager",
+		Version: "v1.13.3",
+		Path:    "https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml",
+	},
 }
 
 func init() {

From 3d6225406d3cb16cdd5c15375045e0558bf8e8aa Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 01:54:34 -0500
Subject: [PATCH 075/113] chore: rename input variable

---
 plugins/scheduler-k3s/template.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index e1a7e36bb..7ac6f356d 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -74,7 +74,7 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 		for _, portMap := range input.PortMaps {
 			ingressRoute := templateKubernetesIngressRoute(IngressRoute{
 				AppName:     input.AppName,
-				Hostnames:   domains,
+				Domains:     domains,
 				Namespace:   input.Namespace,
 				PortMap:     portMap,
 				ProcessType: "web",
@@ -534,7 +534,7 @@ const (
 type IngressRoute struct {
 	AppName     string
 	Entrypoints []IngressRouteEntrypoint
-	Hostnames   []string
+	Domains     []string
 	Namespace   string
 	PortMap     PortMap
 	ProcessType string
@@ -570,7 +570,9 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 		},
 	}
 
-	for _, hostname := range input.Hostnames {
+	sort.Strings(input.Domains)
+
+	for _, hostname := range input.Domains {
 		rule := traefikv1alpha1.Route{
 			Kind:  "Rule",
 			Match: "Host(`" + hostname + "`)",

From b64c3a185901d9b31265bcb7f867c36e268194b9 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 02:10:10 -0500
Subject: [PATCH 076/113] chore: add missing import

---
 plugins/scheduler-k3s/template.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 7ac6f356d..9830163d1 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	appjson "github.com/dokku/dokku/plugins/app-json"

From 2619c3db0804a748e2b186b2088f9cdfa00fcf9e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 05:12:49 -0500
Subject: [PATCH 077/113] fix: ensure critical addons come up when the node is
 tainted

Rather than taint on `node-role.kubernetes.io/master=true:NoSchedule`, we check for `CriticalAddonsOnly=true:NoSchedule` and taint on that.
---
 plugins/scheduler-k3s/functions.go            |  84 ++++++++++--
 plugins/scheduler-k3s/go.mod                  |   1 +
 plugins/scheduler-k3s/go.sum                  |   2 +
 plugins/scheduler-k3s/helm.go                 | 127 +++++++++++++-----
 plugins/scheduler-k3s/k8s.go                  |  17 ++-
 plugins/scheduler-k3s/scheduler_k3s.go        |  71 ++++++++--
 plugins/scheduler-k3s/subcommands.go          |  88 ++++++++----
 .../scheduler-k3s/templates/cert-manager.yaml |  28 ++++
 plugins/scheduler-k3s/templates/longhorn.yaml |  41 ++++++
 .../templates/traefik-config.yaml             |  16 +++
 plugins/scheduler-k3s/triggers.go             |  11 +-
 11 files changed, 400 insertions(+), 86 deletions(-)
 create mode 100644 plugins/scheduler-k3s/templates/cert-manager.yaml
 create mode 100644 plugins/scheduler-k3s/templates/longhorn.yaml
 create mode 100644 plugins/scheduler-k3s/templates/traefik-config.yaml

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index c521f68c7..0eb850257 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -3,6 +3,7 @@ package scheduler_k3s
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"sort"
@@ -11,6 +12,7 @@ import (
 	"time"
 
 	"github.com/dokku/dokku/plugins/common"
+	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -116,17 +118,6 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 		return err
 	}
 
-	namespaces, err := clientset.ListNamespaces(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, namespace := range namespaces {
-		if namespace.Name == namespaceName {
-			return nil
-		}
-	}
-
 	namespace := corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: namespaceName,
@@ -139,7 +130,7 @@ func createKubernetesNamespace(ctx context.Context, namespaceName string) error
 		},
 	}
 	_, err = clientset.CreateNamespace(ctx, CreateNamespaceInput{
-		Namespace: namespace,
+		Name: namespace,
 	})
 	if err != nil {
 		return err
@@ -322,6 +313,73 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	}, nil
 }
 
+func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
+	for _, repo := range HelmRepositories {
+		helmAgent, err := NewHelmAgent("default", DeployLogPrinter)
+		if err != nil {
+			return fmt.Errorf("Error creating helm agent: %w", err)
+		}
+
+		err = helmAgent.AddRepository(ctx, AddRepositoryInput(repo))
+		if err != nil {
+			return fmt.Errorf("Error adding helm repository %s: %w", repo.Name, err)
+		}
+	}
+
+	for _, chart := range HelmCharts {
+		if chart.CreateNamespace {
+			namespace := corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: chart.Namespace,
+					Annotations: map[string]string{
+						"dokku.com/managed": "true",
+					},
+					Labels: map[string]string{
+						"dokku.com/managed": "true",
+					},
+				},
+			}
+			_, err := clientset.CreateNamespace(ctx, CreateNamespaceInput{
+				Name: namespace,
+			})
+			if err != nil {
+				return fmt.Errorf("Error creating namespace %s: %w", chart.Namespace, err)
+			}
+		}
+
+		contents, err := templates.ReadFile(fmt.Sprintf("templates/%s.yaml", chart.ReleaseName))
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err)
+		}
+
+		var values map[string]interface{}
+		if len(contents) > 0 {
+			err = yaml.Unmarshal(contents, &values)
+			if err != nil {
+				return fmt.Errorf("Error unmarshalling values file: %w", err)
+			}
+		}
+
+		helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter)
+		if err != nil {
+			return fmt.Errorf("Error creating helm agent: %w", err)
+		}
+
+		err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
+			ChartPath:   chart.ChartPath,
+			Namespace:   chart.Namespace,
+			ReleaseName: chart.ReleaseName,
+			RepoURL:     chart.RepoURL,
+			Values:      values,
+			Version:     chart.Version,
+		})
+		if err != nil {
+			return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err)
+		}
+	}
+	return nil
+}
+
 func isK3sInstalled() error {
 	if !common.FileExists("/usr/local/bin/k3s") {
 		return fmt.Errorf("k3s binary is not available")
@@ -340,7 +398,7 @@ func isK3sInstalled() error {
 
 func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc {
 	return func(ctx context.Context) (bool, error) {
-		fmt.Printf(".") // progress bar!
+		fmt.Printf(".")
 
 		pod, err := clientset.GetPod(ctx, GetPodInput{
 			Name:      podName,
diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 5ce122cd9..a97118f50 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/fatih/color v1.16.0
 	github.com/go-openapi/jsonpointer v0.19.6
 	github.com/go-resty/resty/v2 v2.11.0
+	github.com/gofrs/flock v0.8.1
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/rancher/wharfie v0.6.5
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index bb66402e8..5d8fd090c 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -155,6 +155,8 @@ github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XE
 github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go
index 79bbfe993..94b176326 100644
--- a/plugins/scheduler-k3s/helm.go
+++ b/plugins/scheduler-k3s/helm.go
@@ -4,18 +4,23 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"os"
-	"os/signal"
+	"path/filepath"
 	"sort"
 	"strings"
-	"syscall"
 	"time"
 	"unicode"
 
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/gofrs/flock"
+	"gopkg.in/yaml.v3"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/kube"
+	"helm.sh/helm/v3/pkg/repo"
 	"helm.sh/helm/v3/pkg/storage/driver"
 )
 
@@ -43,8 +48,10 @@ type ChartInput struct {
 	ChartPath         string
 	Namespace         string
 	ReleaseName       string
+	RepoURL           string
 	RollbackOnFailure bool
 	Timeout           time.Duration
+	Version           string
 	Values            map[string]interface{}
 }
 
@@ -80,7 +87,74 @@ func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error)
 	}, nil
 }
 
+type AddRepositoryInput struct {
+	Name string
+	URL  string
+}
+
+func (h *HelmAgent) AddRepository(ctx context.Context, helmRepo AddRepositoryInput) error {
+	settings := cli.New()
+	repoFile := settings.RepositoryConfig
+
+	err := os.MkdirAll(filepath.Dir(repoFile), os.ModePerm)
+	if err != nil && !os.IsExist(err) {
+		return fmt.Errorf("Error creating repository directory: %w", err)
+	}
+
+	fileLock := flock.New(strings.Replace(repoFile, filepath.Ext(repoFile), ".lock", 1))
+	locked, err := fileLock.TryLockContext(ctx, time.Second)
+	if err != nil {
+		return fmt.Errorf("Error acquiring file lock: %w", err)
+	}
+
+	if !locked {
+		return fmt.Errorf("Could not acquire file lock")
+	}
+
+	defer fileLock.Unlock() // nolint: errcheck
+
+	b, err := os.ReadFile(repoFile)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("Error reading repository file: %w", err)
+	}
+
+	var f repo.File
+	if err := yaml.Unmarshal(b, &f); err != nil {
+		return fmt.Errorf("Error unmarshalling yaml: %w", err)
+	}
+
+	if f.Has(helmRepo.Name) {
+		return nil
+	}
+
+	c := repo.Entry{
+		Name: helmRepo.Name,
+		URL:  helmRepo.URL,
+	}
+
+	r, err := repo.NewChartRepository(&c, getter.All(settings))
+	if err != nil {
+		return fmt.Errorf("Error creating chart repository: %w", err)
+	}
+
+	if _, err := r.DownloadIndexFile(); err != nil {
+		return fmt.Errorf("Specified repository '%q' is not a valid chart repository or cannot be reached: %w", helmRepo.URL, err)
+	}
+
+	f.Update(&c)
+
+	if err := f.WriteFile(repoFile, 0644); err != nil {
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
 func (h *HelmAgent) ChartExists(releaseName string) (bool, error) {
+	if releaseName == "" {
+		return false, fmt.Errorf("Release name is required")
+	}
+
 	client := action.NewHistory(h.Configuration)
 	client.Max = 1
 	releases, err := client.Run(releaseName)
@@ -98,14 +172,14 @@ func (h *HelmAgent) ChartExists(releaseName string) (bool, error) {
 	return false, nil
 }
 
-func (h *HelmAgent) DeleteRevision(releaseName string, revision int) error {
+func (h *HelmAgent) DeleteRevision(ctx context.Context, releaseName string, revision int) error {
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
 	secretName := fmt.Sprintf("sh.helm.release.v1.%s.v%d", releaseName, revision)
-	err = clientset.DeleteSecret(context.Background(), DeleteSecretInput{
+	err = clientset.DeleteSecret(ctx, DeleteSecretInput{
 		Name:      secretName,
 		Namespace: h.Namespace,
 	})
@@ -126,7 +200,7 @@ func (h *HelmAgent) GetValues(releaseName string) (map[string]interface{}, error
 	return values, nil
 }
 
-func (h *HelmAgent) InstallOrUpgradeChart(input ChartInput) error {
+func (h *HelmAgent) InstallOrUpgradeChart(ctx context.Context, input ChartInput) error {
 	chartExists, err := h.ChartExists(input.ReleaseName)
 	if err != nil {
 		return fmt.Errorf("Error checking if chart exists: %w", err)
@@ -134,14 +208,14 @@ func (h *HelmAgent) InstallOrUpgradeChart(input ChartInput) error {
 
 	if chartExists {
 		common.LogExclaim(fmt.Sprintf("Upgrading %s", input.ReleaseName))
-		return h.UpgradeChart(input)
+		return h.UpgradeChart(ctx, input)
 	}
 
 	common.LogExclaim(fmt.Sprintf("Installing %s", input.ReleaseName))
-	return h.InstallChart(input)
+	return h.InstallChart(ctx, input)
 }
 
-func (h *HelmAgent) InstallChart(input ChartInput) error {
+func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error {
 	namespace := input.Namespace
 	if namespace == "" {
 		namespace = h.Namespace
@@ -167,7 +241,15 @@ func (h *HelmAgent) InstallChart(input ChartInput) error {
 	client.Timeout = input.Timeout
 	client.Wait = false
 
-	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil)
+	settings := cli.New()
+	if input.RepoURL != "" {
+		client.ChartPathOptions.RepoURL = input.RepoURL
+	}
+	if input.Version != "" {
+		client.ChartPathOptions.Version = input.Version
+	}
+
+	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, settings)
 	if err != nil {
 		return fmt.Errorf("Error locating chart: %w", err)
 	}
@@ -177,16 +259,6 @@ func (h *HelmAgent) InstallChart(input ChartInput) error {
 		return fmt.Errorf("Error loading chart: %w", err)
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	cSignal := make(chan os.Signal, 2)
-	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-cSignal
-		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", input.ReleaseName))
-		cancel()
-	}()
-
 	_, err = client.RunWithContext(ctx, chartRequested, input.Values)
 	if err != nil {
 		return fmt.Errorf("Error deploying: %w", err)
@@ -195,7 +267,7 @@ func (h *HelmAgent) InstallChart(input ChartInput) error {
 	return nil
 }
 
-func (h *HelmAgent) ListRevisions(releaseName string) ([]Release, error) {
+func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Release, error) {
 	client := action.NewHistory(h.Configuration)
 	response, err := client.Run(releaseName)
 	if err != nil {
@@ -218,7 +290,7 @@ func (h *HelmAgent) ListRevisions(releaseName string) ([]Release, error) {
 	return releases, nil
 }
 
-func (h *HelmAgent) UpgradeChart(input ChartInput) error {
+func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
 	namespace := input.Namespace
 	if namespace == "" {
 		namespace = h.Namespace
@@ -243,6 +315,9 @@ func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 	client.Namespace = namespace
 	client.Timeout = input.Timeout
 	client.Wait = false
+	if input.RepoURL != "" {
+		client.RepoURL = input.RepoURL
+	}
 
 	chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil)
 	if err != nil {
@@ -254,16 +329,6 @@ func (h *HelmAgent) UpgradeChart(input ChartInput) error {
 		return fmt.Errorf("Error loading chart: %w", err)
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	cSignal := make(chan os.Signal, 2)
-	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-cSignal
-		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", input.ReleaseName))
-		cancel()
-	}()
-
 	_, err = client.RunWithContext(ctx, input.ReleaseName, chartRequested, input.Values)
 	if err != nil {
 		return fmt.Errorf("Error deploying: %w", err)
diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go
index 413309f95..8c056581d 100644
--- a/plugins/scheduler-k3s/k8s.go
+++ b/plugins/scheduler-k3s/k8s.go
@@ -154,13 +154,24 @@ func (k KubernetesClient) CreateJob(ctx context.Context, input CreateJobInput) (
 
 // CreateNamespaceInput contains all the information needed to create a Kubernetes namespace
 type CreateNamespaceInput struct {
-	// Namespace is the Kubernetes namespace
-	Namespace v1.Namespace
+	// Name is the name of the Kubernetes namespace
+	Name v1.Namespace
 }
 
 // CreateNamespace creates a Kubernetes namespace
 func (k KubernetesClient) CreateNamespace(ctx context.Context, input CreateNamespaceInput) (v1.Namespace, error) {
-	namespace, err := k.Client.CoreV1().Namespaces().Create(ctx, &input.Namespace, metav1.CreateOptions{})
+	namespaces, err := k.ListNamespaces(ctx)
+	if err != nil {
+		return v1.Namespace{}, err
+	}
+
+	for _, namespace := range namespaces {
+		if namespace.Name == input.Name.Name {
+			return namespace, nil
+		}
+	}
+
+	namespace, err := k.Client.CoreV1().Namespaces().Create(ctx, &input.Name, metav1.CreateOptions{})
 	if err != nil {
 		return v1.Namespace{}, err
 	}
diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index af971b957..13075a584 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -1,6 +1,7 @@
 package scheduler_k3s
 
 import (
+	"embed"
 	"sync"
 
 	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
@@ -49,24 +50,70 @@ type Manifest struct {
 	Path    string
 }
 
-var Manifests = []Manifest{
-	{
-		Name:    "longhorn",
-		Version: "1.5.3",
-		Path:    "https://raw.githubusercontent.com/longhorn/longhorn/v1.5.3/deploy/longhorn.yaml",
-	},
+var KubernetesManifests = []Manifest{
 	{
 		Name:    "system-upgrader",
 		Version: "0.13.2",
-		Path:    "https://github.com/rancher/system-upgrade-controller/releases/v0.13.2/download/system-upgrade-controller.yaml",
-	},
-	{
-		Name:    "cert-manager",
-		Version: "v1.13.3",
-		Path:    "https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml",
+		Path:    "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.2/system-upgrade-controller.yaml",
 	},
 }
 
+type HelmChart struct {
+	ReleaseName     string
+	ChartPath       string
+	Namespace       string
+	CreateNamespace bool
+	Version         string
+	Path            string
+	RepoURL         string
+}
+
+var HelmCharts = []HelmChart{
+	{
+		ChartPath:       "cert-manager",
+		CreateNamespace: true,
+		Namespace:       "cert-manager",
+		ReleaseName:     "cert-manager",
+		RepoURL:         "https://charts.jetstack.io",
+		Version:         "v1.13.3",
+	},
+	{
+		ChartPath:       "longhorn",
+		CreateNamespace: true,
+		Namespace:       "longhorn-system",
+		ReleaseName:     "longhorn",
+		RepoURL:         "https://charts.longhorn.io",
+		Version:         "1.5.3",
+	},
+}
+
+type HelmRepository struct {
+	Name string
+	URL  string
+}
+
+var HelmRepositories = []HelmRepository{
+	{
+		Name: "jetstack",
+		URL:  "https://charts.jetstack.io",
+	},
+	{
+		Name: "longhorn",
+		URL:  "https://charts.longhorn.io",
+	},
+}
+
+var ServerLabels = map[string]string{
+	"svccontroller.k3s.cattle.io/enablelb": "true",
+}
+
+var WorkerLabels = map[string]string{
+	"node-role.kubernetes.io/role": "worker",
+}
+
+//go:embed templates/*
+var templates embed.FS
+
 func init() {
 	k8sNativeSchemeOnce.Do(func() {
 		_ = appsv1.AddToScheme(runtimeScheme)
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 1e94e4311..165591595 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -8,7 +8,9 @@ import (
 	"net"
 	"net/url"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 
 	"github.com/dokku/dokku/plugins/common"
 	resty "github.com/go-resty/resty/v2"
@@ -21,6 +23,17 @@ func CommandInitialize(taintScheduling bool) error {
 		return fmt.Errorf("k3s already installed, cannot re-initialize k3s")
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
 	networkInterface := common.PropertyGetDefault("scheduler-k3s", "--global", "network-interface", "eth0")
 	ifaces, err := net.Interfaces()
 	if err != nil {
@@ -176,7 +189,7 @@ func CommandInitialize(taintScheduling bool) error {
 		"--token", token,
 	}
 	if taintScheduling {
-		args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
+		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
 	}
 
 	common.LogInfo2Quiet("Running k3s installer")
@@ -214,9 +227,13 @@ func CommandInitialize(taintScheduling bool) error {
 	}
 
 	clientset, err := NewKubernetesClient()
-	for _, manifest := range Manifests {
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	for _, manifest := range KubernetesManifests {
 		common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version))
-		err = clientset.ApplyKubernetesManifest(context.Background(), ApplyKubernetesManifestInput{
+		err = clientset.ApplyKubernetesManifest(ctx, ApplyKubernetesManifestInput{
 			Manifest: manifest.Path,
 		})
 		if err != nil {
@@ -224,18 +241,37 @@ func CommandInitialize(taintScheduling bool) error {
 		}
 	}
 
-	common.LogInfo2Quiet("Labeling node svccontroller.k3s.cattle.io/enablelb=true")
-	if err != nil {
-		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	for key, value := range ServerLabels {
+		common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))
+		if err != nil {
+			return fmt.Errorf("Unable to create kubernetes client: %w", err)
+		}
+
+		err = clientset.LabelNode(context.Background(), LabelNodeInput{
+			Name:  nodeName,
+			Key:   key,
+			Value: value,
+		})
+		if err != nil {
+			return fmt.Errorf("Unable to patch node: %w", err)
+		}
 	}
 
-	err = clientset.LabelNode(context.Background(), LabelNodeInput{
-		Name:  nodeName,
-		Key:   "svccontroller.k3s.cattle.io/enablelb",
-		Value: "true",
-	})
+	common.LogInfo2Quiet("Updating traefik config")
+	contents, err := templates.ReadFile("templates/traefik-config.yaml")
 	if err != nil {
-		return fmt.Errorf("Unable to patch node: %w", err)
+		return fmt.Errorf("Unable to read traefik config template: %w", err)
+	}
+
+	err = os.WriteFile("/var/lib/rancher/k3s/server/manifests/traefik-custom.yaml", contents, 0600)
+	if err != nil {
+		return fmt.Errorf("Unable to write traefik config: %w", err)
+	}
+
+	common.LogInfo2Quiet("Installing helm charts")
+	err = installHelmCharts(ctx, clientset)
+	if err != nil {
+		return fmt.Errorf("Unable to install helm charts: %w", err)
 	}
 
 	common.LogVerboseQuiet("Done")
@@ -449,8 +485,9 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		// bind proxy metrics to all interfaces
 		args = append(args, "--kube-proxy-arg", "metrics-bind-address=0.0.0.0")
 	}
+
 	if taintScheduling {
-		args = append(args, "--node-taint", "node-role.kubernetes.io/master=true:NoSchedule")
+		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
 	}
 
 	common.LogInfo2Quiet(fmt.Sprintf("Adding %s k3s cluster", nodeName))
@@ -492,22 +529,21 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		return fmt.Errorf("Error copying registries.yaml to remote host: %w", err)
 	}
 
+	labels := ServerLabels
 	if role == "worker" {
-		common.LogInfo2Quiet("Labeling node kubernetes.io/role=worker")
-		err = clientset.LabelNode(ctx, LabelNodeInput{
-			Name:  nodes[0].Name,
-			Key:   "kubernetes.io/role",
-			Value: "worker",
-		})
+		labels = WorkerLabels
+	}
+
+	for key, value := range labels {
+		common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))
 		if err != nil {
-			return fmt.Errorf("Unable to patch node: %w", err)
+			return fmt.Errorf("Unable to create kubernetes client: %w", err)
 		}
-	} else {
-		common.LogInfo2Quiet("Labeling node svccontroller.k3s.cattle.io/enablelb=true")
-		err = clientset.LabelNode(ctx, LabelNodeInput{
-			Name:  nodes[0].Name,
-			Key:   "svccontroller.k3s.cattle.io/enablelb",
-			Value: "true",
+
+		err = clientset.LabelNode(context.Background(), LabelNodeInput{
+			Name:  nodeName,
+			Key:   key,
+			Value: value,
 		})
 		if err != nil {
 			return fmt.Errorf("Unable to patch node: %w", err)
diff --git a/plugins/scheduler-k3s/templates/cert-manager.yaml b/plugins/scheduler-k3s/templates/cert-manager.yaml
new file mode 100644
index 000000000..b4801fda6
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/cert-manager.yaml
@@ -0,0 +1,28 @@
+installCRDs: true
+
+tolerations:
+- key: CriticalAddonsOnly
+  operator: Equal
+  value: "true"
+  effect: "NoSchedule"
+
+webhook:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+cainjector:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+startupapicheck:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
diff --git a/plugins/scheduler-k3s/templates/longhorn.yaml b/plugins/scheduler-k3s/templates/longhorn.yaml
new file mode 100644
index 000000000..9c274e056
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/longhorn.yaml
@@ -0,0 +1,41 @@
+defaultSettings:
+  taintToleration: "CriticalAddonsOnly=true:NoSchedule"
+
+longhornManager:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornDriver:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornUI:
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornConversionWebhook:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornAdmissionWebhook:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
+
+longhornRecoveryBackend:
+  - key: CriticalAddonsOnly
+    operator: Equal
+    value: "true"
+    effect: "NoSchedule"
diff --git a/plugins/scheduler-k3s/templates/traefik-config.yaml b/plugins/scheduler-k3s/templates/traefik-config.yaml
new file mode 100644
index 000000000..04b95c9b4
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/traefik-config.yaml
@@ -0,0 +1,16 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: traefik-crd
+  namespace: kube-system
+spec:
+  bootstrap: true
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: traefik
+  namespace: kube-system
+spec:
+  bootstrap: true
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 4c0368c50..7d94e52bd 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -167,6 +167,15 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	}
 
 	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	cSignal := make(chan os.Signal, 2)
+	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-cSignal
+		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", appName))
+		cancel()
+	}()
+
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 	if err := createKubernetesNamespace(ctx, namespace); err != nil {
 		return fmt.Errorf("Error creating kubernetes namespace for deployment: %w", err)
@@ -455,7 +464,7 @@ data:
 		return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
 	}
 
-	err = helmAgent.InstallOrUpgradeChart(ChartInput{
+	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
 		ChartPath:         chartPath,
 		Namespace:         namespace,
 		ReleaseName:       fmt.Sprintf("dokku-%s", appName),

From 2bee070e785c48f3c52a6f39b622d38a5b93377d Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 05:34:41 -0500
Subject: [PATCH 078/113] chore: unify context handling

---
 plugins/scheduler-k3s/subcommands.go | 42 ++++++++++++++---
 plugins/scheduler-k3s/triggers.go    | 67 ++++++++++++++++++----------
 2 files changed, 81 insertions(+), 28 deletions(-)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 165591595..1a251a2bc 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -247,7 +247,7 @@ func CommandInitialize(taintScheduling bool) error {
 			return fmt.Errorf("Unable to create kubernetes client: %w", err)
 		}
 
-		err = clientset.LabelNode(context.Background(), LabelNodeInput{
+		err = clientset.LabelNode(ctx, LabelNodeInput{
 			Name:  nodeName,
 			Key:   key,
 			Value: value,
@@ -325,6 +325,17 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		return fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface))
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
 	// todo: check if k3s is installed on the remote host
 
 	k3sVersionCmd, err := common.CallExecCommand(common.ExecCommandInput{
@@ -505,7 +516,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 	if joinCmd.ExitCode != 0 {
 		return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
 	}
-	ctx := context.Background()
+
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
@@ -540,7 +551,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 			return fmt.Errorf("Unable to create kubernetes client: %w", err)
 		}
 
-		err = clientset.LabelNode(context.Background(), LabelNodeInput{
+		err = clientset.LabelNode(ctx, LabelNodeInput{
 			Name:  nodeName,
 			Key:   key,
 			Value: value,
@@ -573,12 +584,23 @@ func CommandClusterList(format string) error {
 		return fmt.Errorf("k3s not installed, cannot list cluster nodes")
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
 	}
 
-	nodes, err := clientset.ListNodes(context.Background(), ListNodesInput{})
+	nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
 	if err != nil {
 		return fmt.Errorf("Unable to list nodes: %w", err)
 	}
@@ -614,6 +636,17 @@ func CommandClusterRemove(nodeName string) error {
 		return fmt.Errorf("k3s not installed, cannot remove node")
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
 	common.LogInfo1Quiet(fmt.Sprintf("Removing %s from k3s cluster", nodeName))
 	clientset, err := NewKubernetesClient()
 	if err != nil {
@@ -621,7 +654,6 @@ func CommandClusterRemove(nodeName string) error {
 	}
 
 	common.LogVerboseQuiet("Getting node remote connection information")
-	ctx := context.Background()
 	node, err := clientset.GetNode(ctx, GetNodeInput{
 		Name: nodeName,
 	})
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 7d94e52bd..0515747ba 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -121,13 +121,22 @@ func TriggerPostRegistryLogin(server string, username string) error {
 		return fmt.Errorf("Unable to write registry configuration to file: %w", err)
 	}
 
-	// todo: auth against all nodes in cluster
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	ctx := context.Background()
 	nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
 	if err != nil {
 		return fmt.Errorf("Error listing nodes: %w", err)
@@ -166,12 +175,14 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return err
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	cSignal := make(chan os.Signal, 2)
-	signal.Notify(cSignal, os.Interrupt, syscall.SIGTERM)
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
 	go func() {
-		<-cSignal
+		<-signals
 		common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", appName))
 		cancel()
 	}()
@@ -505,11 +516,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		return nil
 	}
 
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Error creating kubernetes client: %w", err)
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -521,6 +527,11 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		cancel()
 	}()
 
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 
 	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
@@ -597,11 +608,6 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		return nil
 	}
 
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Error creating kubernetes client: %w", err)
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -613,6 +619,11 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		cancel()
 	}()
 
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
 	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 0
 	if processType != "" {
@@ -1017,6 +1028,17 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er
 		return nil
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM)
+	go func() {
+		<-signals
+		cancel()
+	}()
+
 	clientset, err := NewKubernetesClient()
 	if err != nil {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
@@ -1024,7 +1046,6 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er
 
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 
-	ctx := context.Background()
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
 		Namespace:     namespace,
@@ -1104,11 +1125,6 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 		return nil
 	}
 
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Error creating kubernetes client: %w", err)
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -1120,6 +1136,11 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 		cancel()
 	}()
 
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
 	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
 	deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
 		Namespace:     namespace,

From 01ec08fd7915bf0d08eebc87f2899afe560f4333 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 06:40:04 -0500
Subject: [PATCH 079/113] refactor: simplify port string

---
 plugins/scheduler-k3s/functions.go | 4 ++++
 plugins/scheduler-k3s/template.go  | 8 ++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 0eb850257..3338548bf 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -261,6 +261,10 @@ type PortMap struct {
 	Scheme        string `json:"scheme"`
 }
 
+func (p PortMap) String() string {
+	return fmt.Sprintf("%s-%d-%d", p.Scheme, p.HostPort, p.ContainerPort)
+}
+
 func (p PortMap) IsAllowedHttp() bool {
 	return p.Scheme == "http" || p.ContainerPort == 80
 }
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 9830163d1..edb4e3d8e 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -460,7 +460,7 @@ func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
 				protocol = "UDP"
 			}
 			deployment.Spec.Template.Spec.Containers[0].Ports = append(deployment.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{
-				Name:          fmt.Sprintf("%s-%d-%d", portMap.Scheme, portMap.HostPort, portMap.ContainerPort),
+				Name:          portMap.String(),
 				ContainerPort: portMap.ContainerPort,
 				Protocol:      corev1.Protocol(protocol),
 			})
@@ -557,7 +557,7 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 		"dokku.com/managed": "true",
 	}
 
-	port := fmt.Sprintf("%s-%d-%d", input.PortMap.Scheme, input.PortMap.HostPort, input.PortMap.ContainerPort)
+	port := input.PortMap.String()
 	ingressRoute := traefikv1alpha1.IngressRoute{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        fmt.Sprintf("%s-%s", input.ServiceName, port),
@@ -811,9 +811,9 @@ func templateKubernetesService(input Service) corev1.Service {
 			protocol = "UDP"
 		}
 		service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{
-			Name:       fmt.Sprintf("%s-%d-%d", portMap.Scheme, portMap.HostPort, portMap.ContainerPort),
+			Name:       portMap.String(),
 			Port:       portMap.HostPort,
-			TargetPort: intstr.FromString(fmt.Sprintf("%s-%d-%d", portMap.Scheme, portMap.HostPort, portMap.ContainerPort)),
+			TargetPort: intstr.FromString(portMap.String()),
 			Protocol:   corev1.Protocol(protocol),
 		})
 	}

From b0485495e4660b208b04ea42f359b2869f4ff3bb Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 06:42:03 -0500
Subject: [PATCH 080/113] refactor: make domains a helm value

This will allow updating domains on the fly
---
 plugins/scheduler-k3s/template.go             | 125 +++++++++---------
 .../templates/ingress-routes-append.yaml      |  11 ++
 plugins/scheduler-k3s/triggers.go             |  41 ++++--
 3 files changed, 104 insertions(+), 73 deletions(-)
 create mode 100644 plugins/scheduler-k3s/templates/ingress-routes-append.yaml

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index edb4e3d8e..e10b75c78 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -1,11 +1,11 @@
 package scheduler_k3s
 
 import (
+	"bytes"
 	"crypto/rand"
 	"fmt"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 
 	appjson "github.com/dokku/dokku/plugins/app-json"
@@ -34,63 +34,82 @@ type Chart struct {
 
 type Values struct {
 	DeploymentID string                   `yaml:"deploment_id"`
-	Secrets      map[string]string        `yaml:"secrets"`
-	Processes    map[string]ValuesProcess `yaml:"processes"`
+	Secrets      map[string]string        `yaml:"secrets,omitempty"`
+	Processes    map[string]ProcessValues `yaml:"processes"`
 }
 
-type ValuesProcess struct {
-	Replicas int32 `yaml:"replicas"`
+type ProcessValues struct {
+	Replicas int32    `yaml:"replicas"`
+	Domains  []string `yaml:"domains,omitempty"`
 }
 
 type CreateIngressRoutesInput struct {
-	AppName    string
-	ChartDir   string
-	Deployment appsv1.Deployment
-	Namespace  string
-	PortMaps   []PortMap
-	Service    v1.Service
+	AppName     string
+	ChartDir    string
+	Deployment  appsv1.Deployment
+	Namespace   string
+	ProcessType string
+	PortMaps    []PortMap
+	Service     v1.Service
 }
 
 func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
-	err := common.PlugnTrigger("domains-vhost-enabled", []string{input.AppName}...)
-	isAppVhostEnabled := err == nil
+	for _, portMap := range input.PortMaps {
+		ingressRoute := templateKubernetesIngressRoute(IngressRoute{
+			AppName:     input.AppName,
+			Namespace:   input.Namespace,
+			PortMap:     portMap,
+			ProcessType: input.ProcessType,
+			ServiceName: input.Service.Name,
+		})
 
-	if isAppVhostEnabled {
-		b, err := common.PlugnTriggerOutput("domains-list", []string{input.AppName}...)
+		ingressRouteFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/ingress-route-%s.yaml", portMap.String()))
+		err := writeResourceToFile(WriteResourceInput{
+			Object: &ingressRoute,
+			Path:   ingressRouteFile,
+		})
 		if err != nil {
-			return fmt.Errorf("Error getting domains for deployment: %w", err)
-		}
-		domains := []string{}
-		for _, domain := range strings.Split(string(b), "\n") {
-			domain = strings.TrimSpace(domain)
-			if domain != "" {
-				domains = append(domains, domain)
-			}
+			return fmt.Errorf("Error printing ingress route: %w", err)
 		}
 
-		if len(domains) == 0 {
-			return nil
+		b, err := os.ReadFile(ingressRouteFile)
+		if err != nil {
+			return fmt.Errorf("Error reading ingress route file: %w", err)
 		}
 
-		for _, portMap := range input.PortMaps {
-			ingressRoute := templateKubernetesIngressRoute(IngressRoute{
-				AppName:     input.AppName,
-				Domains:     domains,
-				Namespace:   input.Namespace,
-				PortMap:     portMap,
-				ProcessType: "web",
-				ServiceName: input.Service.Name,
-			})
+		append, err := templates.ReadFile("templates/ingress-routes-append.yaml")
+		if err != nil {
+			return fmt.Errorf("Error reading ingress route append file: %w", err)
+		}
 
-			err = writeResourceToFile(WriteResourceInput{
-				Object: &ingressRoute,
-				Path:   filepath.Join(input.ChartDir, fmt.Sprintf("templates/ingress-route-%s-%d-%d.yaml", portMap.Scheme, portMap.HostPort, portMap.ContainerPort)),
-			})
-			if err != nil {
-				return fmt.Errorf("Error printing ingress route: %w", err)
-			}
+		contents := string(bytes.Join([][]byte{b, append}, []byte("")))
+		contents = strings.ReplaceAll(contents, "  routes: null", "")
+		contents = strings.Join([]string{
+			"{{- if .Values.processes.PROCESS_TYPE.domains }}",
+			contents,
+			"{{- end }}",
+		}, "\n")
+
+		replacements := orderedmap.New[string, string]()
+		replacements.Set("PROCESS_TYPE", input.ProcessType)
+		replacements.Set("APP_NAME", input.AppName)
+		replacements.Set("NAMESPACE", input.Namespace)
+		replacements.Set("PORT_MAPPING", portMap.String())
+		replacements.Set("PORT_SCHEME", portMap.Scheme)
+		for pair := replacements.Oldest(); pair != nil; pair = pair.Next() {
+			contents = strings.ReplaceAll(contents, pair.Key, pair.Value)
+		}
+
+		err = os.WriteFile(ingressRouteFile, []byte(contents), os.FileMode(0644))
+		if err != nil {
+			return fmt.Errorf("Error writing ingress route file: %w", err)
+		}
+
+		if os.Getenv("DOKKU_TRACE") == "1" {
+			common.CatFile(ingressRouteFile)
 		}
 	}
+
 	return nil
 }
 
@@ -535,7 +554,6 @@ const (
 type IngressRoute struct {
 	AppName     string
 	Entrypoints []IngressRouteEntrypoint
-	Domains     []string
 	Namespace   string
 	PortMap     PortMap
 	ProcessType string
@@ -567,32 +585,9 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 		},
 		Spec: traefikv1alpha1.IngressRouteSpec{
 			EntryPoints: []string{string(entryPoint)},
-			Routes:      []traefikv1alpha1.Route{},
 		},
 	}
 
-	sort.Strings(input.Domains)
-
-	for _, hostname := range input.Domains {
-		rule := traefikv1alpha1.Route{
-			Kind:  "Rule",
-			Match: "Host(`" + hostname + "`)",
-			Services: []traefikv1alpha1.Service{
-				{
-					LoadBalancerSpec: traefikv1alpha1.LoadBalancerSpec{
-						Name:           input.ServiceName,
-						Namespace:      input.Namespace,
-						PassHostHeader: ptr.To(true),
-						Port:           intstr.FromString(port),
-						Scheme:         input.PortMap.Scheme,
-					},
-				},
-			},
-		}
-
-		ingressRoute.Spec.Routes = append(ingressRoute.Spec.Routes, rule)
-	}
-
 	return ingressRoute
 }
 
diff --git a/plugins/scheduler-k3s/templates/ingress-routes-append.yaml b/plugins/scheduler-k3s/templates/ingress-routes-append.yaml
new file mode 100644
index 000000000..a4b0a1e98
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/ingress-routes-append.yaml
@@ -0,0 +1,11 @@
+  routes:
+    {{- range .Values.processes.PROCESS_TYPE.domains }}
+    - kind: Rule
+      match: Host(`{{ . }}`)
+      services:
+      - name: APP_NAME-PROCESS_TYPE
+        namespace: NAMESPACE
+        passHostHeader: true
+        port: PORT_MAPPING
+        scheme: PORT_SCHEME
+    {{- end }}
\ No newline at end of file
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 0515747ba..7eb1022aa 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"syscall"
@@ -394,6 +395,7 @@ data:
 		}
 	}
 
+	domains := []string{}
 	if deployment, ok := deployments["web"]; ok {
 		service := templateKubernetesService(Service{
 			AppName:   appName,
@@ -410,13 +412,29 @@ data:
 			return fmt.Errorf("Error printing service: %w", err)
 		}
 
+		err = common.PlugnTrigger("domains-vhost-enabled", []string{appName}...)
+		if err == nil {
+			b, err := common.PlugnTriggerOutput("domains-list", []string{appName}...)
+			if err != nil {
+				return fmt.Errorf("Error getting domains for deployment: %w", err)
+			}
+
+			for _, domain := range strings.Split(string(b), "\n") {
+				domain = strings.TrimSpace(domain)
+				if domain != "" {
+					domains = append(domains, domain)
+				}
+			}
+		}
+
 		err = createIngressRoutesFiles(CreateIngressRoutesInput{
-			AppName:    appName,
-			ChartDir:   chartDir,
-			Deployment: deployment,
-			Namespace:  namespace,
-			PortMaps:   portMaps,
-			Service:    service,
+			AppName:     appName,
+			ChartDir:    chartDir,
+			Deployment:  deployment,
+			Namespace:   namespace,
+			PortMaps:    portMaps,
+			ProcessType: "web",
+			Service:     service,
 		})
 		if err != nil {
 			return fmt.Errorf("Error creating ingress routes: %w", err)
@@ -441,13 +459,20 @@ data:
 	values := &Values{
 		DeploymentID: fmt.Sprint(deploymentId),
 		Secrets:      map[string]string{},
-		Processes:    map[string]ValuesProcess{},
+		Processes:    map[string]ProcessValues{},
 	}
 	for processType, processCount := range processes {
-		values.Processes[processType] = ValuesProcess{
+		processValues := ProcessValues{
 			Replicas: int32(processCount),
 		}
+		if processType == "web" {
+			sort.Strings(domains)
+			processValues.Domains = domains
+		}
+
+		values.Processes[processType] = processValues
 	}
+
 	for key, value := range env.Map() {
 		values.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value))
 	}

From 63a6e9087f1af368243bd3a430addef6cb567923 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 07:18:20 -0500
Subject: [PATCH 081/113] feat: create tls certificate if tls is enabled and
 the app has domains

---
 plugins/scheduler-k3s/go.mod                  |  16 +-
 plugins/scheduler-k3s/go.sum                  |  40 ++--
 plugins/scheduler-k3s/scheduler_k3s.go        |   2 +
 plugins/scheduler-k3s/template.go             | 172 +++++++++++++++++-
 .../templates/certificate-append.yaml         |   4 +
 plugins/scheduler-k3s/triggers.go             |  19 ++
 6 files changed, 227 insertions(+), 26 deletions(-)
 create mode 100644 plugins/scheduler-k3s/templates/certificate-append.yaml

diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index a97118f50..32de4e5dc 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -3,6 +3,7 @@ module github.com/dokku/dokku/plugins/scheduler-k3s
 go 1.21
 
 require (
+	github.com/cert-manager/cert-manager v1.13.3
 	github.com/dokku/dokku/plugins/app-json v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
@@ -66,7 +67,7 @@ require (
 	github.com/go-acme/lego/v4 v4.14.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
-	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.1 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -143,15 +144,15 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
-	go.opentelemetry.io/otel v1.19.0 // indirect
-	go.opentelemetry.io/otel/metric v1.19.0 // indirect
-	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	go.opentelemetry.io/otel v1.20.0 // indirect
+	go.opentelemetry.io/otel/metric v1.20.0 // indirect
+	go.opentelemetry.io/otel/trace v1.20.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
@@ -159,8 +160,8 @@ require (
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
-	google.golang.org/grpc v1.58.3 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect
+	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -170,6 +171,7 @@ require (
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	oras.land/oras-go v1.2.4 // indirect
+	sigs.k8s.io/gateway-api v0.8.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index 5d8fd090c..42f1c5ca9 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -52,6 +52,8 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3k
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cert-manager/cert-manager v1.13.3 h1:3R4G0RI7K0OkTZhWlVOC5SGZMYa2NwqmQJoyKydrz/M=
+github.com/cert-manager/cert-manager v1.13.3/go.mod h1:BM2+Pt/NmSv1Zr25/MHv6BgIEF9IUxA1xAjp80qkxgc=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
@@ -112,8 +114,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
-github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
 github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
 github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
@@ -124,8 +126,8 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs=
 github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw=
-github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
-github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
+github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@@ -383,8 +385,8 @@ github.com/rancher/wharfie v0.6.5 h1:Xxj/ki8d4BFeQBvyViW9jX8BqxaPlZwXvRD8BG2Mm/8
 github.com/rancher/wharfie v0.6.5/go.mod h1:iGXn5Y9GEhsx1h/lPUXAklK4DoY4+6NOJF+Li1j248Y=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
-github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97 h1:3RPlVWzZ/PDqmVuf/FKHARG5EMid/tl7cv54Sw/QRVY=
-github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=
 github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -450,12 +452,12 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
-go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
-go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
-go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
-go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
-go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
+go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc=
+go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs=
+go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA=
+go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM=
+go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ=
+go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -503,8 +505,8 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
+golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -583,13 +585,13 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 h1:o4LtQxebKIJ4vkzyhtD2rfUNZ20Zf0ik5YVP5E7G7VE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
-google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -652,6 +654,8 @@ mvdan.cc/sh/v3 v3.7.0 h1:lSTjdP/1xsddtaKfGg7Myu7DnlHItd3/M2tomOcNNBg=
 mvdan.cc/sh/v3 v3.7.0/go.mod h1:K2gwkaesF/D7av7Kxl0HbF5kGOd2ArupNTX3X44+8l8=
 oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=
 oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
+sigs.k8s.io/gateway-api v0.8.0 h1:isQQ3Jx2qFP7vaA3ls0846F0Amp9Eq14P08xbSwVbQg=
+sigs.k8s.io/gateway-api v0.8.0/go.mod h1:okOnjPNBFbIS/Rw9kAhuIUaIkLhTKEu+ARIuXk2dgaM=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 13075a584..86250bfb2 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -4,6 +4,7 @@ import (
 	"embed"
 	"sync"
 
+	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -118,6 +119,7 @@ func init() {
 	k8sNativeSchemeOnce.Do(func() {
 		_ = appsv1.AddToScheme(runtimeScheme)
 		_ = batchv1.AddToScheme(runtimeScheme)
+		_ = certmanagerv1.AddToScheme(runtimeScheme)
 		_ = corev1.AddToScheme(runtimeScheme)
 		_ = traefikv1alpha1.AddToScheme(runtimeScheme)
 	})
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index e10b75c78..e2338ec86 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -8,6 +8,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	acmev1 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
+	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
+	certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
 	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
 	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
@@ -39,8 +42,9 @@ type Values struct {
 }
 
 type ProcessValues struct {
-	Replicas int32    `yaml:"replicas"`
 	Domains  []string `yaml:"domains,omitempty"`
+	Replicas int32    `yaml:"replicas"`
+	TLS      bool     `yaml:"tls"`
 }
 
 type CreateIngressRoutesInput struct {
@@ -113,6 +117,172 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 	return nil
 }
 
+type CreateCertificateFileInput struct {
+	Certificate Certificate
+	ChartDir    string
+	IssuerName  string
+	ProcessType string
+}
+
+type Certificate struct {
+	AppName   string
+	Name      string
+	Namespace string
+	TLS       bool
+}
+
+func createCertificateFile(input CreateCertificateFileInput) error {
+	certificate, err := templateKubernetesCertificate(input.Certificate)
+	if err != nil {
+		return fmt.Errorf("Error templating certificate: %w", err)
+	}
+
+	certificateFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/certificate-%s.yaml", input.ProcessType))
+	err = writeResourceToFile(WriteResourceInput{
+		Object: &certificate,
+		Path:   certificateFile,
+	})
+	if err != nil {
+		return fmt.Errorf("Error printing ingress route: %w", err)
+	}
+
+	b, err := os.ReadFile(certificateFile)
+	if err != nil {
+		return fmt.Errorf("Error reading ingress route file: %w", err)
+	}
+
+	append, err := templates.ReadFile("templates/certificate-append.yaml")
+	if err != nil {
+		return fmt.Errorf("Error reading ingress route append file: %w", err)
+	}
+
+	contents := string(bytes.Join([][]byte{b, append}, []byte("")))
+	contents = strings.Join([]string{
+		"{{- if and .Values.processes.PROCESS_TYPE.tls .Values.processes.PROCESS_TYPE.domains }}",
+		contents,
+		"{{- end }}",
+	}, "\n")
+
+	replacements := orderedmap.New[string, string]()
+	replacements.Set("PROCESS_TYPE", input.ProcessType)
+	replacements.Set("ISSUER_NAME", input.IssuerName)
+	for pair := replacements.Oldest(); pair != nil; pair = pair.Next() {
+		contents = strings.ReplaceAll(contents, pair.Key, pair.Value)
+	}
+
+	err = os.WriteFile(certificateFile, []byte(contents), os.FileMode(0644))
+	if err != nil {
+		return fmt.Errorf("Error writing ingress route file: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(certificateFile)
+	}
+
+	return nil
+}
+
+func templateKubernetesCertificate(input Certificate) (certmanagerv1.Certificate, error) {
+	if input.Name == "" {
+		return certmanagerv1.Certificate{}, fmt.Errorf("Name cannot be empty")
+	}
+	if input.Namespace == "" {
+		return certmanagerv1.Certificate{}, fmt.Errorf("Namespace cannot be empty")
+	}
+
+	labels := map[string]string{
+		"app.kubernetes.io/name":    input.Name,
+		"app.kubernetes.io/part-of": input.AppName,
+	}
+	annotations := map[string]string{
+		"dokku.com/managed": "true",
+	}
+
+	certificate := certmanagerv1.Certificate{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        input.Name,
+			Namespace:   input.Namespace,
+			Labels:      labels,
+			Annotations: annotations,
+		},
+		Spec: certmanagerv1.CertificateSpec{
+			SecretName: "tls-" + input.Name,
+			SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
+				Annotations: annotations,
+				Labels:      labels,
+			},
+			IssuerRef: certmanagermetav1.ObjectReference{
+				Name: "ISSUER_NAME",
+				Kind: "ClusterIssuer",
+			},
+		},
+	}
+
+	return certificate, nil
+}
+
+type ClusterIssuer struct {
+	Email        string
+	IngressClass string
+	Name         string
+	Namespace    string
+	Server       string
+}
+
+func templateKubernetesClusterIssuer(input ClusterIssuer) (certmanagerv1.ClusterIssuer, error) {
+	if input.Email == "" {
+		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Email cannot be empty")
+	}
+	if input.Server == "" {
+		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Server cannot be empty")
+	}
+	if input.Name == "" {
+		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Name cannot be empty")
+	}
+	if input.Namespace == "" {
+		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Namespace cannot be empty")
+	}
+
+	if input.Server == "staging" {
+		input.Server = "https://acme-staging-v02.api.letsencrypt.org/directory"
+	} else if input.Server == "production" {
+		input.Server = "https://acme-v02.api.letsencrypt.org/directory"
+	} else {
+		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Server must be either staging or production")
+	}
+
+	clusterIssuer := certmanagerv1.ClusterIssuer{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      input.Name,
+			Namespace: input.Namespace,
+		},
+		Spec: certmanagerv1.IssuerSpec{
+			IssuerConfig: certmanagerv1.IssuerConfig{
+				ACME: &acmev1.ACMEIssuer{
+					Email:  input.Email,
+					Server: input.Server,
+					PrivateKey: certmanagermetav1.SecretKeySelector{
+						LocalObjectReference: certmanagermetav1.LocalObjectReference{
+							Name: input.Name,
+						},
+					},
+					Solvers: []acmev1.ACMEChallengeSolver{
+						{
+							HTTP01: &acmev1.ACMEChallengeSolverHTTP01{
+								Ingress: &acmev1.ACMEChallengeSolverHTTP01Ingress{
+									Class: ptr.To(input.IngressClass),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	return clusterIssuer, nil
+}
+
 type Job struct {
 	AppName          string
 	Command          []string
diff --git a/plugins/scheduler-k3s/templates/certificate-append.yaml b/plugins/scheduler-k3s/templates/certificate-append.yaml
new file mode 100644
index 000000000..538ddfdaa
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/certificate-append.yaml
@@ -0,0 +1,4 @@
+  dnsNames:
+    {{- range .Values.processes.PROCESS_TYPE.domains }}
+    - {{ . }}
+    {{- end }}
\ No newline at end of file
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 7eb1022aa..bd1fb202a 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -395,6 +395,9 @@ data:
 		}
 	}
 
+	// todo: make this configurable
+	tls := false
+
 	domains := []string{}
 	if deployment, ok := deployments["web"]; ok {
 		service := templateKubernetesService(Service{
@@ -439,6 +442,21 @@ data:
 		if err != nil {
 			return fmt.Errorf("Error creating ingress routes: %w", err)
 		}
+
+		err = createCertificateFile(CreateCertificateFileInput{
+			ChartDir: chartDir,
+			Certificate: Certificate{
+				AppName:   appName,
+				Name:      fmt.Sprintf("%s-%s", appName, "web"),
+				Namespace: namespace,
+				TLS:       tls,
+			},
+			IssuerName:  "letsencrypt-prod",
+			ProcessType: "web",
+		})
+		if err != nil {
+			return fmt.Errorf("Error creating certificate files: %w", err)
+		}
 	}
 
 	chart := &Chart{
@@ -468,6 +486,7 @@ data:
 		if processType == "web" {
 			sort.Strings(domains)
 			processValues.Domains = domains
+			processValues.TLS = tls
 		}
 
 		values.Processes[processType] = processValues

From 29760e8ecdd1493f276832ea3c08a5fd2c94b139 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 07:36:42 -0500
Subject: [PATCH 082/113] feat: add support to tls for ingress-route

If tls is enabled on the process, we create add the https entrypoint and also add the requisite tls secret-name
---
 plugins/scheduler-k3s/template.go                        | 9 ++-------
 .../scheduler-k3s/templates/ingress-routes-append.yaml   | 9 ++++++++-
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index e2338ec86..3ea21ef1f 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -86,7 +86,7 @@ func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
 			return fmt.Errorf("Error reading ingress route append file: %w", err)
 		}
 
-		contents := string(bytes.Join([][]byte{b, append}, []byte("")))
+		contents := strings.Join([]string{strings.TrimSpace(string(b)), string(append)}, "")
 		contents = strings.ReplaceAll(contents, "  routes: null", "")
 		contents = strings.Join([]string{
 			"{{- if .Values.processes.PROCESS_TYPE.domains }}",
@@ -731,11 +731,6 @@ type IngressRoute struct {
 }
 
 func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressRoute {
-	entryPoint := IngressRouteEntrypoint_HTTP
-	if input.PortMap.Scheme == "https" {
-		entryPoint = IngressRouteEntrypoint_HTTPS
-	}
-
 	labels := map[string]string{
 		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
 		"app.kubernetes.io/name":     input.ProcessType,
@@ -754,7 +749,7 @@ func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressR
 			Annotations: annotations,
 		},
 		Spec: traefikv1alpha1.IngressRouteSpec{
-			EntryPoints: []string{string(entryPoint)},
+			EntryPoints: []string{string(IngressRouteEntrypoint_HTTP)},
 		},
 	}
 
diff --git a/plugins/scheduler-k3s/templates/ingress-routes-append.yaml b/plugins/scheduler-k3s/templates/ingress-routes-append.yaml
index a4b0a1e98..237ebeef9 100644
--- a/plugins/scheduler-k3s/templates/ingress-routes-append.yaml
+++ b/plugins/scheduler-k3s/templates/ingress-routes-append.yaml
@@ -1,3 +1,6 @@
+  {{ if .Values.processes.PROCESS_TYPE.tls }}
+  - websecure
+  {{- end }}
   routes:
     {{- range .Values.processes.PROCESS_TYPE.domains }}
     - kind: Rule
@@ -8,4 +11,8 @@
         passHostHeader: true
         port: PORT_MAPPING
         scheme: PORT_SCHEME
-    {{- end }}
\ No newline at end of file
+    {{- end }}
+  {{ if .Values.processes.PROCESS_TYPE.tls }}
+  tls:
+    secretName: tls-APP_NAME-PROCESS_TYPE
+  {{- end }}

From 661dfbbf517d3297e80484a36c6842a5640daaa6 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 07:38:59 -0500
Subject: [PATCH 083/113] chore: ignore yaml files in scheduler-k3s plugin

---
 .github/linters/.yamllint.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/linters/.yamllint.yml b/.github/linters/.yamllint.yml
index 75da2b702..6c02c589d 100644
--- a/.github/linters/.yamllint.yml
+++ b/.github/linters/.yamllint.yml
@@ -3,3 +3,6 @@ extends: default
 
 rules:
   line-length: disable
+
+ignore:
+  - plugins/scheduler-k3s/templates/*
\ No newline at end of file

From 22f27ea2d36d125f34b09d5ae98b972c775d5d6c Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 07:39:52 -0500
Subject: [PATCH 084/113] fix: add missing newline

---
 .github/linters/.yamllint.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/linters/.yamllint.yml b/.github/linters/.yamllint.yml
index 6c02c589d..fe6352dcc 100644
--- a/.github/linters/.yamllint.yml
+++ b/.github/linters/.yamllint.yml
@@ -5,4 +5,4 @@ rules:
   line-length: disable
 
 ignore:
-  - plugins/scheduler-k3s/templates/*
\ No newline at end of file
+  - plugins/scheduler-k3s/templates/*

From 62c8cb770eba20372215b8a3fdb54f636f15fc59 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 08:01:11 -0500
Subject: [PATCH 085/113] feat: add support for global properties

---
 plugins/scheduler-k3s/functions.go     | 120 ++++++++++++++++++-------
 plugins/scheduler-k3s/portmap.go       |  52 +++++++++++
 plugins/scheduler-k3s/scheduler_k3s.go |  17 ++--
 plugins/scheduler-k3s/subcommands.go   |   8 +-
 plugins/scheduler-k3s/triggers.go      |  24 +++--
 5 files changed, 166 insertions(+), 55 deletions(-)
 create mode 100644 plugins/scheduler-k3s/portmap.go

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 3338548bf..2cf64ba9f 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -2,7 +2,6 @@ package scheduler_k3s
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -255,48 +254,105 @@ func extractStartCommand(input StartCommandInput) string {
 	return command
 }
 
-type PortMap struct {
-	ContainerPort int32  `json:"container_port"`
-	HostPort      int32  `json:"host_port"`
-	Scheme        string `json:"scheme"`
+func getDeployTimeout(appName string) string {
+	return common.PropertyGetDefault("scheduler-k3s", appName, "deploy-timeout", "")
 }
 
-func (p PortMap) String() string {
-	return fmt.Sprintf("%s-%d-%d", p.Scheme, p.HostPort, p.ContainerPort)
+func getGlobalDeployTimeout() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "deploy-timeout", "300s")
 }
 
-func (p PortMap) IsAllowedHttp() bool {
-	return p.Scheme == "http" || p.ContainerPort == 80
-}
-
-func (p PortMap) IsAllowedHttps() bool {
-	return p.Scheme == "https" || p.ContainerPort == 443
-}
-
-func getPortMaps(appName string) ([]PortMap, error) {
-	portMaps := []PortMap{}
-
-	output, err := common.PlugnTriggerOutputAsString("ports-get", []string{appName, "json"}...)
-	if err != nil {
-		return portMaps, err
+func getComputedDeployTimeout(appName string) string {
+	deployTimeout := getDeployTimeout(appName)
+	if deployTimeout == "" {
+		deployTimeout = getGlobalDeployTimeout()
 	}
 
-	err = json.Unmarshal([]byte(output), &portMaps)
-	if err != nil {
-		return portMaps, err
+	return deployTimeout
+}
+
+func getImagePullSecrets(appName string) string {
+	return common.PropertyGetDefault("scheduler-k3s", appName, "image-pull-secrets", "")
+}
+
+func getGlobalImagePullSecrets() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "image-pull-secrets", "")
+}
+
+func getComputedImagePullSecrets(appName string) string {
+	imagePullSecrets := getImagePullSecrets(appName)
+	if imagePullSecrets == "" {
+		imagePullSecrets = getGlobalImagePullSecrets()
 	}
 
-	allowedMappings := []PortMap{}
-	for _, portMap := range portMaps {
-		if !portMap.IsAllowedHttp() && !portMap.IsAllowedHttps() {
-			// todo: log warning
-			continue
-		}
+	return imagePullSecrets
+}
 
-		allowedMappings = append(allowedMappings, portMap)
+func getLetsencryptServer(appName string) string {
+	return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "")
+}
+
+func getGlobalLetsencryptServer(appName string) string {
+	return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "prod")
+}
+
+func getComputedLetsencryptServer(appName string) string {
+	letsencryptServer := getLetsencryptServer(appName)
+	if letsencryptServer == "" {
+		letsencryptServer = getGlobalLetsencryptServer(appName)
 	}
 
-	return allowedMappings, nil
+	return letsencryptServer
+}
+
+func getGlobalLetsencryptEmailProd() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "letsencrypt-email-prod", "")
+}
+
+func getGlobalLetsencryptEmailStag() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "letsencrypt-email-stag", "")
+}
+
+func getNamespace(appName string) string {
+	return common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "")
+}
+
+func getGlobalNamespace() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "namespace", "default")
+}
+
+func getComputedNamespace(appName string) string {
+	namespace := getNamespace(appName)
+	if namespace == "" {
+		namespace = getGlobalNamespace()
+	}
+
+	return namespace
+}
+
+func getGlobalNetworkInterface() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "network-interface", "eth0")
+}
+
+func getRollbackOnFailure(appName string) string {
+	return common.PropertyGetDefault("scheduler-k3s", appName, "rollback-on-failure", "")
+}
+
+func getGlobalRollbackOnFailure() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "rollback-on-failure", "")
+}
+
+func getComputedRollbackOnFailure(appName string) string {
+	rollbackOnFailure := getRollbackOnFailure(appName)
+	if rollbackOnFailure == "" {
+		rollbackOnFailure = getGlobalRollbackOnFailure()
+	}
+
+	return rollbackOnFailure
+}
+
+func getGlobalGlobalToken() string {
+	return common.PropertyGet("scheduler-k3s", "--global", "token")
 }
 
 func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
diff --git a/plugins/scheduler-k3s/portmap.go b/plugins/scheduler-k3s/portmap.go
new file mode 100644
index 000000000..e4aa03807
--- /dev/null
+++ b/plugins/scheduler-k3s/portmap.go
@@ -0,0 +1,52 @@
+package scheduler_k3s
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+type PortMap struct {
+	ContainerPort int32  `json:"container_port"`
+	HostPort      int32  `json:"host_port"`
+	Scheme        string `json:"scheme"`
+}
+
+func (p PortMap) String() string {
+	return fmt.Sprintf("%s-%d-%d", p.Scheme, p.HostPort, p.ContainerPort)
+}
+
+func (p PortMap) IsAllowedHttp() bool {
+	return p.Scheme == "http" || p.ContainerPort == 80
+}
+
+func (p PortMap) IsAllowedHttps() bool {
+	return p.Scheme == "https" || p.ContainerPort == 443
+}
+
+func getPortMaps(appName string) ([]PortMap, error) {
+	portMaps := []PortMap{}
+
+	output, err := common.PlugnTriggerOutputAsString("ports-get", []string{appName, "json"}...)
+	if err != nil {
+		return portMaps, err
+	}
+
+	err = json.Unmarshal([]byte(output), &portMaps)
+	if err != nil {
+		return portMaps, err
+	}
+
+	allowedMappings := []PortMap{}
+	for _, portMap := range portMaps {
+		if !portMap.IsAllowedHttp() && !portMap.IsAllowedHttps() {
+			// todo: log warning
+			continue
+		}
+
+		allowedMappings = append(allowedMappings, portMap)
+	}
+
+	return allowedMappings, nil
+}
diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 86250bfb2..5ec4631ca 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -17,18 +17,23 @@ import (
 var (
 	// DefaultProperties is a map of all valid k3s properties with corresponding default property values
 	DefaultProperties = map[string]string{
+		"deploy-timeout":      "",
+		"letsencrypt-server":  "",
 		"image-pull-secrets":  "",
+		"namespace":           "",
 		"rollback-on-failure": "",
-		"namespace":           "default",
-		"timeout":             "300",
 	}
 
 	// GlobalProperties is a map of all valid global k3s properties
 	GlobalProperties = map[string]bool{
-		"network-interface":   true,
-		"rollback-on-failure": true,
-		"token":               true,
-		"timeout":             true,
+		"deploy-timeout":         true,
+		"image-pull-secrets":     true,
+		"letsencrypt-email-prod": true,
+		"letsencrypt-email-stag": true,
+		"namespace":              true,
+		"network-interface":      true,
+		"rollback-on-failure":    true,
+		"token":                  true,
 	}
 )
 
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 1a251a2bc..8faeac9fa 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -34,7 +34,7 @@ func CommandInitialize(taintScheduling bool) error {
 		cancel()
 	}()
 
-	networkInterface := common.PropertyGetDefault("scheduler-k3s", "--global", "network-interface", "eth0")
+	networkInterface := getGlobalNetworkInterface()
 	ifaces, err := net.Interfaces()
 	if err != nil {
 		return fmt.Errorf("Unable to get network interfaces: %w", err)
@@ -142,7 +142,7 @@ func CommandInitialize(taintScheduling bool) error {
 		return fmt.Errorf("Invalid k3s installer filesize")
 	}
 
-	token := common.PropertyGet("scheduler-k3s", "--global", "token")
+	token := getGlobalGlobalToken()
 	if len(token) == 0 {
 		n := 5
 		b := make([]byte, n)
@@ -289,7 +289,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		return fmt.Errorf("Invalid server-type: %s", role)
 	}
 
-	token := common.PropertyGet("scheduler-k3s", "--global", "token")
+	token := getGlobalGlobalToken()
 	if len(token) == 0 {
 		return fmt.Errorf("Missing k3s token")
 	}
@@ -298,7 +298,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		return fmt.Errorf("Taint scheduling can only be used on the server role")
 	}
 
-	networkInterface := common.PropertyGetDefault("scheduler-k3s", "--global", "network-interface", "eth0")
+	networkInterface := getGlobalNetworkInterface()
 	ifaces, err := net.Interfaces()
 	if err != nil {
 		return fmt.Errorf("Unable to get network interfaces: %w", err)
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index bd1fb202a..9c4b8e6c6 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -188,7 +188,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		cancel()
 	}()
 
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	namespace := getComputedNamespace(appName)
 	if err := createKubernetesNamespace(ctx, namespace); err != nil {
 		return fmt.Errorf("Error creating kubernetes namespace for deployment: %w", err)
 	}
@@ -198,18 +198,18 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error getting deploying app image name: %w", err)
 	}
 
-	deployTimeout := common.PropertyGetDefault("scheduler-k3s", appName, "deploy-timeout", "300s")
+	deployTimeout := getComputedDeployTimeout(appName)
 	if _, err := strconv.Atoi(deployTimeout); err == nil {
 		deployTimeout = fmt.Sprintf("%ss", deployTimeout)
 	}
 
-	deployRollback := common.PropertyGetDefault("scheduler-k3s", appName, "rollback-on-failure", "false")
+	deployRollback := getComputedRollbackOnFailure(appName)
 	allowRollbacks, err := strconv.ParseBool(deployRollback)
 	if err != nil {
 		return fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err)
 	}
 
-	imagePullSecrets := common.PropertyGetDefault("scheduler-k3s", appName, "image-pull-secrets", "")
+	imagePullSecrets := getComputedImagePullSecrets(appName)
 
 	imageSourceType := "dockerfile"
 	if common.IsImageCnbBased(image) {
@@ -576,8 +576,7 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
-
+	namespace := getComputedNamespace(appName)
 	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 1
 	if processType != "" {
@@ -682,7 +681,7 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		labelSelector = append(labelSelector, fmt.Sprintf("app.kubernetes.io/name=%s", processType))
 	}
 
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	namespace := getComputedNamespace(appName)
 	pods, err := clientset.ListPods(ctx, ListPodsInput{
 		Namespace:     namespace,
 		LabelSelector: strings.Join(labelSelector, ","),
@@ -881,7 +880,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		entrypoint = "/exec"
 	}
 
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	namespace := getComputedNamespace(appName)
 	helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
 	if err != nil {
 		return fmt.Errorf("Error creating helm agent: %w", err)
@@ -905,7 +904,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	}
 
 	attachToPod := os.Getenv("DOKKU_DETACH_CONTAINER") != "1"
-	imagePullSecrets := common.PropertyGetDefault("scheduler-k3s", appName, "image-pull-secrets", "")
+	imagePullSecrets := getComputedImagePullSecrets(appName)
 	workingDir := common.GetWorkingDir(appName, image)
 	job, err := templateKubernetesJob(Job{
 		AppName:          appName,
@@ -1088,8 +1087,7 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
-
+	namespace := getComputedNamespace(appName)
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
 		Namespace:     namespace,
@@ -1149,7 +1147,7 @@ func TriggerSchedulerPostDelete(scheduler string, appName string) error {
 	if scheduler != "k3s" {
 		return nil
 	}
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	namespace := getComputedNamespace(appName)
 	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
 	if err != nil {
 		return fmt.Errorf("Error creating helm agent: %w", err)
@@ -1185,7 +1183,7 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	namespace := common.PropertyGetDefault("scheduler-k3s", appName, "namespace", "default")
+	namespace := getComputedNamespace(appName)
 	deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
 		Namespace:     namespace,
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),

From b1802547ae2e4c947d4c08685e66ef4086cd7dca Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 08:01:33 -0500
Subject: [PATCH 086/113] feat: add letsencrypt support

---
 plugins/scheduler-k3s/triggers.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 9c4b8e6c6..fea7c18ac 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -395,8 +395,21 @@ data:
 		}
 	}
 
-	// todo: make this configurable
+	issuerName := "letsencrypt-stag"
+	server := getComputedLetsencryptServer(appName)
+	if server == "prod" || server == "production" {
+		issuerName = "letsencrypt-prod"
+	} else if server != "stag" && server != "staging" {
+		return fmt.Errorf("Invalid letsencrypt server config: %s", server)
+	}
+
 	tls := false
+	if issuerName == "letsencrypt-stag" {
+		tls = getGlobalLetsencryptEmailStag() != ""
+	}
+	if issuerName == "letsencrypt-prod" {
+		tls = getGlobalLetsencryptEmailProd() != ""
+	}
 
 	domains := []string{}
 	if deployment, ok := deployments["web"]; ok {
@@ -451,7 +464,7 @@ data:
 				Namespace: namespace,
 				TLS:       tls,
 			},
-			IssuerName:  "letsencrypt-prod",
+			IssuerName:  issuerName,
 			ProcessType: "web",
 		})
 		if err != nil {

From 001ae3c83b82c6135b426d98613d473cc2e10863 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 21:14:55 -0500
Subject: [PATCH 087/113] fix: set default value for rollback-on-failure

---
 plugins/scheduler-k3s/functions.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 2cf64ba9f..9f43ec8bc 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -339,7 +339,7 @@ func getRollbackOnFailure(appName string) string {
 }
 
 func getGlobalRollbackOnFailure() string {
-	return common.PropertyGetDefault("scheduler-k3s", "--global", "rollback-on-failure", "")
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "rollback-on-failure", "false")
 }
 
 func getComputedRollbackOnFailure(appName string) string {

From 2eb346d50d08d11ea21a0a7b0dc64bf4f55aa1e2 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 21:15:21 -0500
Subject: [PATCH 088/113] fix: allow setting letsencrypt-server globally

---
 plugins/scheduler-k3s/scheduler_k3s.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go
index 5ec4631ca..bf3e6f5e7 100644
--- a/plugins/scheduler-k3s/scheduler_k3s.go
+++ b/plugins/scheduler-k3s/scheduler_k3s.go
@@ -28,6 +28,7 @@ var (
 	GlobalProperties = map[string]bool{
 		"deploy-timeout":         true,
 		"image-pull-secrets":     true,
+		"letsencrypt-server":     true,
 		"letsencrypt-email-prod": true,
 		"letsencrypt-email-stag": true,
 		"namespace":              true,

From 75539273e3b8c88e5dbfcfe7bc3d737591ea7c89 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Mon, 22 Jan 2024 21:15:29 -0500
Subject: [PATCH 089/113] docs: update documentation

---
 docs/deployment/schedulers/k3s.md | 245 ++++++++++++++++++++++++++++--
 1 file changed, 231 insertions(+), 14 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index 0ee1d1fce..c39076768 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -22,6 +22,234 @@ As it is the default, unsetting the `selected` scheduler property is also a vali
 dokku scheduler:set node-js-app k3s
 ```
 
+## Usage
+
+### Initializing a cluster
+
+> [!WARNING]
+> This command must be run as root
+
+Clusters can be initialized via the `scheduler-k3s:initialize` command. This will start a k3s cluster on the Dokku node itself.
+
+```shell
+dokku scheduler-k3s:initialize
+```
+
+By default, the k3s installation can run both app and system workloads. For clusters where app workloads are run on distinct worker nodes, initialize the cluster with the `--taint-scheduling` flag, which will allow _only_ Critical cluster components on the k3s control-plane nodes.
+
+```shell
+dokku scheduler-k3s:initialize --taint-scheduling
+```
+
+### Adding nodes to the cluster
+
+> [!WARNING]
+> The `dokku` user _must_ be able to ssh onto the server in order to connect nodes to the cluster. The remote user must be root or have sudo enabled, or the install will fail.
+
+#### Adding a worker node
+
+Nodes that run app workloads can be added via the `scheduler-k3s:cluster-add` command. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in worker mode. Workers are typically used to run app workloads.
+
+```shell
+dokku scheduler-k3s:cluster-add  ssh://root@worker-1.example.com
+```
+
+If the server isn't in the `known_hosts` file, the connection will fail. This can be bypassed by setting the `--insecure-allow-unknown-hosts` flag:
+
+```shell
+dokku scheduler-k3s:cluster-add --insecure-allow-unknown-hosts ssh://root@worker-1.example.com
+```
+
+#### Adding a server node
+
+> [!NOTE]
+> Only the initial Dokku server will be properly configured for push deployment, and should be considered your git remote. Additional server nodes are for ensuring high-availability of the K3s etcd state. Ensure this server is properly backed up and restorable or deployments will not work.
+
+Server nodes are typically used to replicate the cluster state, and it is recommended to have an odd number of nodes spread across several availability zones (datacenters in close proximity within a region). This allows for higher availability in the event of a cluster failure. Server nodes run control-plane services such as the traefik load balancer and the etcd backing store.
+
+Server nodes can also be added with the `scheduler-k3s:cluster-add` command by specifying `--role server`. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in server mode.
+
+```shell
+dokku scheduler-k3s:cluster-add  --role server ssh://root@server-1.example.com
+```
+
+Server nodes allow any workloads to be scheduled on them by default, in addition to the control-plane, etcd, and the scheduler itself. To avoid app workloads being scheduled on your control-plane, use the `--taint-scheduling` flag:
+
+```shell
+dokku scheduler-k3s:cluster-add --role server --taint-scheduling ssh://root@cluster-1.example.com
+```
+
+If the server isn't in the `known_hosts` file, the connection will fail. This can be bypassed by setting the `--insecure-allow-unknown-hosts` flag:
+
+```shell
+dokku scheduler-k3s:cluster-add --role server --insecure-allow-unknown-hosts ssh://root@worker-1.example.com
+```
+
+#### Changing the network interface
+
+When attaching an worker or server node, the K3s plugin will look at the IP associated with the `eth0` interface and use that to connect the new node to the cluster. To change this, set the `network-interface` property to the appropriate value.
+
+```shell
+dokku scheduler-k3s:set --global network-interface eth1
+```
+
+### Changing deploy timeouts
+
+By default, app deploys will timeout after 300s. To customize this value, set the `deploy-timeout` property via `scheduler-k3s:set`:
+
+```shell
+dokku scheduler-k3s:set node-js-app deploy-timeout 60s
+```
+
+The default value may be set by passing an empty value for the option:
+
+```shell
+dokku scheduler-k3s:set node-js-app deploy-timeout
+```
+
+The `deploy-timeout` property can also be set globally. The global default is `300s`.
+
+```shell
+dokku scheduler-k3s:set --global deploy-timeout 60s
+```
+
+The default value may be set by passing an empty value for the option.
+
+```shell
+dokku scheduler-k3s:set --global deploy-timeout
+```
+
+### Customizing the namespace
+
+By default, app deploys will run against the `default` Kubernetes namespace. To customize this value, set the `namespace` property via `scheduler-k3s:set`:
+
+```shell
+dokku scheduler-k3s:set node-js-app namespace lollipop
+```
+
+The default value may be set by passing an empty value for the option:
+
+```shell
+dokku scheduler-k3s:set node-js-app namespace
+```
+
+The `namespace` property can also be set globally. The global default is `default`.
+
+```shell
+dokku scheduler-k3s:set --global namespace 60s
+```
+
+The default value may be set by passing an empty value for the option.
+
+```shell
+dokku scheduler-k3s:set --global namespace
+```
+
+### Enabling rollback on failure
+
+By default, app deploys do not rollback on failure. To enable this functionality, set the `rollback-on-failure` property via `scheduler-k3s:set`:
+
+```shell
+dokku scheduler-k3s:set node-js-app rollback-on-failure true
+```
+
+The default value may be set by passing an empty value for the option:
+
+```shell
+dokku scheduler-k3s:set node-js-app rollback-on-failure
+```
+
+The `rollback-on-failure` property can also be set globally. The global default is `false`.
+
+```shell
+dokku scheduler-k3s:set --global rollback-on-failure false
+```
+
+The default value may be set by passing an empty value for the option.
+
+```shell
+dokku scheduler-k3s:set --global rollback-on-failure
+```
+
+### Using image pull secrets
+
+When authenticating against a registry via `registry:login`, the scheduler-k3s plugin will authenticate all servers in the cluster against the registry specified. If desired, an image pull secret can be used instead. To customize this value, set the `image-pull-secrets` property via `scheduler-k3s:set`:
+
+```shell
+dokku scheduler-k3s:set node-js-app image-pull-secrets lollipop
+```
+
+The default value may be set by passing an empty value for the option:
+
+```shell
+dokku scheduler-k3s:set node-js-app image-pull-secrets
+```
+
+The `image-pull-secrets` property can also be set globally. The global default is empty string, and k3s will use the local `registries.yaml` for any private registry pulls.
+
+```shell
+dokku scheduler-k3s:set --global image-pull-secrets 60s
+```
+
+The default value may be set by passing an empty value for the option.
+
+```shell
+dokku scheduler-k3s:set --global image-pull-secrets
+```
+
+#### Enabling letsencrypt integration
+
+By default, letsencrypt is disabled and https port mappings are ignored. To enable, set the `letsencrypt-email-prod` or `letsencrypt-email-stag` property with the `--global` flag:
+
+```shell
+# set the value for prod
+dokku scheduler-k3s:set --global letsencrypt-email-prod automated@dokku.sh
+
+# set the value for stag
+dokku scheduler-k3s:set --global letsencrypt-email-stag automated@dokku.sh
+```
+
+After enabling, all apps with an `https` port mapping that utilize the related letsencrypt server will be automatically updated to enable ssl. All http requests will then be redirected to https.
+
+#### Customizing the letsencrypt server
+
+The letsencrypt integration is set to the production letsencrypt server by default. This can be changed on an app-level by setting the `letsencrypt-server` property with the `scheduler-k3s:set` command
+
+```shell
+dokku scheduler-k3s:set node-js-app letsencrypt-server staging
+```
+
+The default value may be set by passing an empty value for the option:
+
+```shell
+dokku scheduler-k3s:set node-js-app letsencrypt-server
+```
+
+The `image-pull-secrets` property can also be set globally. The global default is `production`.
+
+```shell
+dokku scheduler-k3s:set --global letsencrypt-server staging
+```
+
+The default value may be set by passing an empty value for the option.
+
+```shell
+dokku scheduler-k3s:set --global letsencrypt-server staging
+```
+
+After changing, all apps with an 
+
+### Using kubectl remotely
+
+> [!WARNING]
+> Certain ports must be open for interacting with the remote kubernets api. Refer to the [K3s networking documentation](https://docs.k3s.io/installation/requirements?os=debian#networking) for the required open ports between servers prior to running the command.
+
+By default, Dokku assumes that all it controls all actions on the cluster, and thus does not expose the `kubectl` binary for administrators. To interact with kubectl, you will need to retrieve the `kubeconfig` for the cluster and configure your client to use that configuration.
+
+```shell
+dokku scheduler-k3s:show-kubeconfig
+```
+
 ## Tutorial
 
 ### Single-node cluster
@@ -105,9 +333,9 @@ At this point, all app deploys will be performed against the k3s cluster.
 > [!WARNING]
 > Certain ports must be open for cross-server communication. Refer to the [K3s networking documentation](https://docs.k3s.io/installation/requirements?os=debian#networking) for the required open ports between servers prior to running the command.
 
-For high-availability, it is recommended to add both agent and server nodes to the cluster. Dokku will default to starting the cluster with an embedded Etcd database backend, and is ready to add new agent or server nodes immediately.
+For high-availability, it is recommended to add both worker and server nodes to the cluster. Dokku will default to starting the cluster with an embedded Etcd database backend, and is ready to add new worker or server nodes immediately.
 
-When attaching an agent or server node, the K3s plugin will look at the IP associated with the `eth0` interface and use that to connect the new node to the cluster. To change this, set the `network-interface` property to the appropriate value.
+When attaching an worker or server node, the K3s plugin will look at the IP associated with the `eth0` interface and use that to connect the new node to the cluster. To change this, set the `network-interface` property to the appropriate value.
 
 ```shell
 dokku scheduler-k3s:set --global network-interface eth1
@@ -148,23 +376,12 @@ Server nodes are typically used to replicate the cluster state, and it is recomm
 > [!NOTE]
 > Only the initial Dokku server will be properly configured for push deployment, and should be considered your git remote. Additional server nodes are for ensuring high-availability of the K3s etcd state. Ensure this server is properly backed up and restorable or deployments will not work.
 
-Agent nodes are used to run. To add an agent, run the `scheduler-k3s:cluster-add` with the `--role worker` flag. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in agent mode. Agents are typically used to run app workloads.
+Worker nodes are used to run. To add an worker, run the `scheduler-k3s:cluster-add` with the `--role worker` flag. This will ssh onto the specified server, install k3s, and join it to the current Dokku node in worker mode. Workers are typically used to run app workloads.
 
 ```shell
 dokku scheduler-k3s:cluster-add --role worker ssh://root@worker-1.example.com
 ```
 
-### Using kubectl remotely
-
-> [!WARNING]
-> Certain ports must be open for interacting with the remote kubernets api. Refer to the [K3s networking documentation](https://docs.k3s.io/installation/requirements?os=debian#networking) for the required open ports between servers prior to running the command.
-
-By default, Dokku assumes that all it controls all actions on the cluster, and thus does not expose the `kubectl` binary for administrators. To interact with kubectl, you will need to retrieve the `kubeconfig` for the cluster and configure your client to use that configuration.
-
-```shell
-dokku scheduler-k3s:show-kubeconfig
-```
-
 ## Scheduler Interface
 
 The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler.

From 2336063ca213a29f47abe42df0377ab7e8372f9e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 01:29:50 -0500
Subject: [PATCH 090/113] refactor: move templating into pseudo-generated helm
 chart

The previous mechanism of constructing the helm chart was pretty obnoxious due to needing to have some things in the object and some in template files that were appended afterwards. This change simplifies that by using template files per type with minimal replacements.
---
 plugins/scheduler-k3s/functions.go            | 149 ++-
 plugins/scheduler-k3s/subcommands.go          |   2 +-
 plugins/scheduler-k3s/template.go             | 874 +++---------------
 .../templates/certificate-append.yaml         |   4 -
 .../templates/chart/certificate.yaml          |  32 +
 .../templates/chart/cron-job.yaml             | 103 +++
 .../templates/chart/deployment.yaml           |  96 ++
 .../templates/chart/ingress-route.yaml        |  35 +
 .../scheduler-k3s/templates/chart/secret.yaml |  16 +
 .../templates/chart/service.yaml              |  26 +
 .../{ => helm-config}/cert-manager.yaml       |   0
 .../templates/{ => helm-config}/longhorn.yaml |   0
 .../{ => helm-config}/traefik-config.yaml     |   0
 .../templates/ingress-routes-append.yaml      |  18 -
 plugins/scheduler-k3s/triggers.go             | 367 ++++----
 15 files changed, 752 insertions(+), 970 deletions(-)
 delete mode 100644 plugins/scheduler-k3s/templates/certificate-append.yaml
 create mode 100644 plugins/scheduler-k3s/templates/chart/certificate.yaml
 create mode 100644 plugins/scheduler-k3s/templates/chart/cron-job.yaml
 create mode 100644 plugins/scheduler-k3s/templates/chart/deployment.yaml
 create mode 100644 plugins/scheduler-k3s/templates/chart/ingress-route.yaml
 create mode 100644 plugins/scheduler-k3s/templates/chart/secret.yaml
 create mode 100644 plugins/scheduler-k3s/templates/chart/service.yaml
 rename plugins/scheduler-k3s/templates/{ => helm-config}/cert-manager.yaml (100%)
 rename plugins/scheduler-k3s/templates/{ => helm-config}/longhorn.yaml (100%)
 rename plugins/scheduler-k3s/templates/{ => helm-config}/traefik-config.yaml (100%)
 delete mode 100644 plugins/scheduler-k3s/templates/ingress-routes-append.yaml

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 9f43ec8bc..e7387bbac 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -10,10 +10,12 @@ import (
 	"strings"
 	"time"
 
+	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
 	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -355,6 +357,151 @@ func getGlobalGlobalToken() string {
 	return common.PropertyGet("scheduler-k3s", "--global", "token")
 }
 
+func getProcessHealtchecks(healthchecks []appjson.Healthcheck, primaryPort int32) ProcessHealthchecks {
+	if len(healthchecks) == 0 {
+		return ProcessHealthchecks{}
+	}
+
+	livenessChecks := []ProcessHealthcheck{}
+	readinessChecks := []ProcessHealthcheck{}
+	startupChecks := []ProcessHealthcheck{}
+	uptimeSeconds := []int32{}
+	for _, healthcheck := range healthchecks {
+		probe := ProcessHealthcheck{
+			InitialDelaySeconds: healthcheck.InitialDelay,
+			PeriodSeconds:       healthcheck.Wait,
+			TimeoutSeconds:      healthcheck.Timeout,
+			FailureThreshold:    healthcheck.Attempts,
+			SuccessThreshold:    int32(1),
+		}
+		if len(healthcheck.Command) > 0 {
+			probe.Exec = &ExecHealthcheck{
+				Command: healthcheck.Command,
+			}
+		} else if healthcheck.Listening {
+			probe.TCPSocket = &TCPHealthcheck{
+				Port: primaryPort,
+			}
+			for _, header := range healthcheck.HTTPHeaders {
+				if header.Name == "Host" {
+					probe.TCPSocket.Host = header.Value
+				}
+			}
+		} else if healthcheck.Path != "" {
+			probe.HTTPGet = &HTTPHealthcheck{
+				Path:        healthcheck.Path,
+				Port:        primaryPort,
+				HTTPHeaders: []HTTPHeader{},
+			}
+
+			if healthcheck.Scheme != "" {
+				probe.HTTPGet.Scheme = URIScheme(strings.ToUpper(healthcheck.Scheme))
+			}
+
+			for _, header := range healthcheck.HTTPHeaders {
+				probe.HTTPGet.HTTPHeaders = append(probe.HTTPGet.HTTPHeaders, HTTPHeader{
+					Name:  header.Name,
+					Value: header.Value,
+				})
+			}
+		} else if healthcheck.Uptime > 0 {
+			uptimeSeconds = append(uptimeSeconds, healthcheck.Uptime)
+		}
+
+		if healthcheck.Type == appjson.HealthcheckType_Liveness {
+			livenessChecks = append(livenessChecks, probe)
+		} else if healthcheck.Type == appjson.HealthcheckType_Readiness {
+			readinessChecks = append(readinessChecks, probe)
+		} else if healthcheck.Type == appjson.HealthcheckType_Startup {
+			startupChecks = append(startupChecks, probe)
+		}
+	}
+	if len(livenessChecks) > 1 {
+		common.LogWarn("Multiple liveness checks are not supported, only the first one will be used")
+	}
+	if len(readinessChecks) > 1 {
+		common.LogWarn("Multiple readiness checks are not supported, only the first one will be used")
+	}
+	if len(startupChecks) > 1 {
+		common.LogWarn("Multiple startup checks are not supported, only the first one will be used")
+	}
+	if len(uptimeSeconds) > 1 {
+		common.LogWarn("Multiple uptime checks are not supported, only the first one will be used")
+	}
+
+	processHealthchecks := ProcessHealthchecks{}
+	if len(livenessChecks) > 0 {
+		processHealthchecks.Liveness = livenessChecks[0]
+	}
+	if len(readinessChecks) > 0 {
+		processHealthchecks.Readiness = readinessChecks[0]
+	}
+	if len(startupChecks) > 0 {
+		processHealthchecks.Startup = startupChecks[0]
+	}
+	if len(uptimeSeconds) > 0 {
+		processHealthchecks.MinReadySeconds = uptimeSeconds[0]
+	}
+
+	return processHealthchecks
+}
+
+func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) {
+	processResources := ProcessResourcesMap{
+		Limits: ProcessResources{
+			CPU:    "1000m",
+			Memory: "512Mi",
+		},
+		Requests: ProcessResources{
+			CPU:    "1000m",
+			Memory: "512Mi",
+		},
+	}
+	cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "cpu"}...)
+	if err != nil && cpuLimit != "" && cpuLimit != "0" {
+		_, err := resource.ParseQuantity(cpuLimit)
+		if err != nil {
+			return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err)
+		}
+		processResources.Limits.CPU = cpuLimit
+	}
+	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...)
+	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
+		_, err := resource.ParseQuantity(nvidiaGpuLimit)
+		if err != nil {
+			return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
+		}
+		processResources.Limits.NvidiaGPU = nvidiaGpuLimit
+	}
+	memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "memory"}...)
+	if err != nil && memoryLimit != "" && memoryLimit != "0" {
+		_, err := resource.ParseQuantity(memoryLimit)
+		if err != nil {
+			return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err)
+		}
+		processResources.Limits.Memory = memoryLimit
+	}
+
+	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "cpu"}...)
+	if err != nil && cpuRequest != "" && cpuRequest != "0" {
+		_, err := resource.ParseQuantity(cpuRequest)
+		if err != nil {
+			return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err)
+		}
+		processResources.Requests.CPU = cpuRequest
+	}
+	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "memory"}...)
+	if err != nil && memoryRequest != "" && memoryRequest != "0" {
+		_, err := resource.ParseQuantity(memoryRequest)
+		if err != nil {
+			return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err)
+		}
+		processResources.Requests.Memory = memoryRequest
+	}
+
+	return processResources, nil
+}
+
 func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	command := extractStartCommand(input)
 	fields, err := shell.Fields(command, func(name string) string {
@@ -407,7 +554,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
 			}
 		}
 
-		contents, err := templates.ReadFile(fmt.Sprintf("templates/%s.yaml", chart.ReleaseName))
+		contents, err := templates.ReadFile(fmt.Sprintf("templates/helm-config/%s.yaml", chart.ReleaseName))
 		if err != nil && !errors.Is(err, os.ErrNotExist) {
 			return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err)
 		}
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 8faeac9fa..ee1a2b0c2 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -258,7 +258,7 @@ func CommandInitialize(taintScheduling bool) error {
 	}
 
 	common.LogInfo2Quiet("Updating traefik config")
-	contents, err := templates.ReadFile("templates/traefik-config.yaml")
+	contents, err := templates.ReadFile("templates/helm-config/traefik-config.yaml")
 	if err != nil {
 		return fmt.Errorf("Unable to read traefik config template: %w", err)
 	}
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 3ea21ef1f..845ebe3e7 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -1,30 +1,20 @@
 package scheduler_k3s
 
 import (
-	"bytes"
 	"crypto/rand"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	acmev1 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
 	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
-	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
-	traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
-	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
-	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/cli-runtime/pkg/printers"
 	"k8s.io/utils/ptr"
 )
 
@@ -36,189 +26,141 @@ type Chart struct {
 }
 
 type Values struct {
-	DeploymentID string                   `yaml:"deploment_id"`
-	Secrets      map[string]string        `yaml:"secrets,omitempty"`
-	Processes    map[string]ProcessValues `yaml:"processes"`
+	Global    GlobalValues             `yaml:"global"`
+	Processes map[string]ProcessValues `yaml:"processes"`
+}
+
+type GlobalValues struct {
+	AppName      string            `yaml:"app_name"`
+	DeploymentID string            `yaml:"deploment_id"`
+	Image        GlobalImage       `yaml:"image"`
+	Namespace    string            `yaml:"namespace"`
+	PrimaryPort  int32             `yaml:"primary_port"`
+	Secrets      map[string]string `yaml:"secrets,omitempty"`
+}
+
+type GlobalImage struct {
+	ImagePullSecrets string `yaml:"image_pull_secrets"`
+	Name             string `yaml:"name"`
+	Type             string `yaml:"type"`
+	WorkingDir       string `yaml:"working_dir"`
 }
 
 type ProcessValues struct {
-	Domains  []string `yaml:"domains,omitempty"`
-	Replicas int32    `yaml:"replicas"`
-	TLS      bool     `yaml:"tls"`
+	Args         []string            `yaml:"args,omitempty"`
+	Cron         ProcessCron         `yaml:"cron,omitempty"`
+	Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"`
+	ProcessType  ProcessType         `yaml:"process_type"`
+	Replicas     int32               `yaml:"replicas"`
+	Resources    ProcessResourcesMap `yaml:"resources,omitempty"`
+	Web          ProcessWeb          `yaml:"web,omitempty"`
 }
 
-type CreateIngressRoutesInput struct {
-	AppName     string
-	ChartDir    string
-	Deployment  appsv1.Deployment
-	Namespace   string
-	ProcessType string
-	PortMaps    []PortMap
-	Service     v1.Service
+type ProcessHealthchecks struct {
+	Liveness        ProcessHealthcheck `yaml:"liveness,omitempty"`
+	Readiness       ProcessHealthcheck `yaml:"readiness,omitempty"`
+	Startup         ProcessHealthcheck `yaml:"startup,omitempty"`
+	MinReadySeconds int32              `yaml:"min_ready_seconds,omitempty"`
 }
 
-func createIngressRoutesFiles(input CreateIngressRoutesInput) error {
-	for _, portMap := range input.PortMaps {
-		ingressRoute := templateKubernetesIngressRoute(IngressRoute{
-			AppName:     input.AppName,
-			Namespace:   input.Namespace,
-			PortMap:     portMap,
-			ProcessType: input.ProcessType,
-			ServiceName: input.Service.Name,
-		})
+type ProcessHealthcheck struct {
+	Exec      *ExecHealthcheck `yaml:"exec,omitempty"`
+	HTTPGet   *HTTPHealthcheck `yaml:"httpGet,omitempty"`
+	TCPSocket *TCPHealthcheck  `yaml:"tcpSocket,omitempty"`
 
-		ingressRouteFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/ingress-route-%s.yaml", portMap.String()))
-		err := writeResourceToFile(WriteResourceInput{
-			Object: &ingressRoute,
-			Path:   ingressRouteFile,
-		})
-		if err != nil {
-			return fmt.Errorf("Error printing ingress route: %w", err)
-		}
-
-		b, err := os.ReadFile(ingressRouteFile)
-		if err != nil {
-			return fmt.Errorf("Error reading ingress route file: %w", err)
-		}
-
-		append, err := templates.ReadFile("templates/ingress-routes-append.yaml")
-		if err != nil {
-			return fmt.Errorf("Error reading ingress route append file: %w", err)
-		}
-
-		contents := strings.Join([]string{strings.TrimSpace(string(b)), string(append)}, "")
-		contents = strings.ReplaceAll(contents, "  routes: null", "")
-		contents = strings.Join([]string{
-			"{{- if .Values.processes.PROCESS_TYPE.domains }}",
-			contents,
-			"{{- end }}",
-		}, "\n")
-
-		replacements := orderedmap.New[string, string]()
-		replacements.Set("PROCESS_TYPE", input.ProcessType)
-		replacements.Set("APP_NAME", input.AppName)
-		replacements.Set("NAMESPACE", input.Namespace)
-		replacements.Set("PORT_MAPPING", portMap.String())
-		replacements.Set("PORT_SCHEME", portMap.Scheme)
-		for pair := replacements.Oldest(); pair != nil; pair = pair.Next() {
-			contents = strings.ReplaceAll(contents, pair.Key, pair.Value)
-		}
-
-		err = os.WriteFile(ingressRouteFile, []byte(contents), os.FileMode(0644))
-		if err != nil {
-			return fmt.Errorf("Error writing ingress route file: %w", err)
-		}
-
-		if os.Getenv("DOKKU_TRACE") == "1" {
-			common.CatFile(ingressRouteFile)
-		}
-	}
-
-	return nil
+	InitialDelaySeconds           int32  `yaml:"initialDelaySeconds,omitempty"`
+	TimeoutSeconds                int32  `yaml:"timeoutSeconds,omitempty"`
+	PeriodSeconds                 int32  `yaml:"periodSeconds,omitempty"`
+	SuccessThreshold              int32  `yaml:"successThreshold,omitempty"`
+	FailureThreshold              int32  `yaml:"failureThreshold,omitempty"`
+	TerminationGracePeriodSeconds *int64 `yaml:"terminationGracePeriodSeconds,omitempty"`
 }
 
-type CreateCertificateFileInput struct {
-	Certificate Certificate
-	ChartDir    string
-	IssuerName  string
-	ProcessType string
+type ExecHealthcheck struct {
+	Command []string `yaml:"command,omitempty"`
 }
 
-type Certificate struct {
-	AppName   string
-	Name      string
-	Namespace string
-	TLS       bool
+type HTTPHealthcheck struct {
+	Path        string       `yaml:"path,omitempty"`
+	Port        int32        `yaml:"port,omitempty"`
+	Host        string       `yaml:"host,omitempty"`
+	Scheme      URIScheme    `yaml:"scheme,omitempty"`
+	HTTPHeaders []HTTPHeader `yaml:"httpHeaders,omitempty"`
 }
 
-func createCertificateFile(input CreateCertificateFileInput) error {
-	certificate, err := templateKubernetesCertificate(input.Certificate)
-	if err != nil {
-		return fmt.Errorf("Error templating certificate: %w", err)
-	}
-
-	certificateFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/certificate-%s.yaml", input.ProcessType))
-	err = writeResourceToFile(WriteResourceInput{
-		Object: &certificate,
-		Path:   certificateFile,
-	})
-	if err != nil {
-		return fmt.Errorf("Error printing ingress route: %w", err)
-	}
-
-	b, err := os.ReadFile(certificateFile)
-	if err != nil {
-		return fmt.Errorf("Error reading ingress route file: %w", err)
-	}
-
-	append, err := templates.ReadFile("templates/certificate-append.yaml")
-	if err != nil {
-		return fmt.Errorf("Error reading ingress route append file: %w", err)
-	}
-
-	contents := string(bytes.Join([][]byte{b, append}, []byte("")))
-	contents = strings.Join([]string{
-		"{{- if and .Values.processes.PROCESS_TYPE.tls .Values.processes.PROCESS_TYPE.domains }}",
-		contents,
-		"{{- end }}",
-	}, "\n")
-
-	replacements := orderedmap.New[string, string]()
-	replacements.Set("PROCESS_TYPE", input.ProcessType)
-	replacements.Set("ISSUER_NAME", input.IssuerName)
-	for pair := replacements.Oldest(); pair != nil; pair = pair.Next() {
-		contents = strings.ReplaceAll(contents, pair.Key, pair.Value)
-	}
-
-	err = os.WriteFile(certificateFile, []byte(contents), os.FileMode(0644))
-	if err != nil {
-		return fmt.Errorf("Error writing ingress route file: %w", err)
-	}
-
-	if os.Getenv("DOKKU_TRACE") == "1" {
-		common.CatFile(certificateFile)
-	}
-
-	return nil
+type TCPHealthcheck struct {
+	Port int32  `yaml:"port,omitempty"`
+	Host string `yaml:"host,omitempty"`
 }
 
-func templateKubernetesCertificate(input Certificate) (certmanagerv1.Certificate, error) {
-	if input.Name == "" {
-		return certmanagerv1.Certificate{}, fmt.Errorf("Name cannot be empty")
-	}
-	if input.Namespace == "" {
-		return certmanagerv1.Certificate{}, fmt.Errorf("Namespace cannot be empty")
-	}
+type HTTPHeader struct {
+	Name  string `yaml:"name"`
+	Value string `yaml:"value"`
+}
 
-	labels := map[string]string{
-		"app.kubernetes.io/name":    input.Name,
-		"app.kubernetes.io/part-of": input.AppName,
-	}
-	annotations := map[string]string{
-		"dokku.com/managed": "true",
-	}
+type URIScheme string
 
-	certificate := certmanagerv1.Certificate{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        input.Name,
-			Namespace:   input.Namespace,
-			Labels:      labels,
-			Annotations: annotations,
-		},
-		Spec: certmanagerv1.CertificateSpec{
-			SecretName: "tls-" + input.Name,
-			SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
-				Annotations: annotations,
-				Labels:      labels,
-			},
-			IssuerRef: certmanagermetav1.ObjectReference{
-				Name: "ISSUER_NAME",
-				Kind: "ClusterIssuer",
-			},
-		},
-	}
+const (
+	URISchemeHTTP  URIScheme = "HTTP"
+	URISchemeHTTPS URIScheme = "HTTPS"
+)
 
-	return certificate, nil
+type ProcessWeb struct {
+	Domains  []string         `yaml:"domains,omitempty"`
+	PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"`
+	TLS      bool             `yaml:"tls"`
+}
+
+type ProcessResourcesMap struct {
+	Limits   ProcessResources `yaml:"limits,omitempty"`
+	Requests ProcessResources `yaml:"requests,omitempty"`
+}
+
+type ProcessResources struct {
+	NvidiaGPU string `yaml:"nvidia.com/gpu,omitempty"`
+	CPU       string `yaml:"cpu,omitempty"`
+	Memory    string `yaml:"memory,omitempty"`
+}
+
+type ProcessType string
+
+const (
+	ProcessType_Cron   ProcessType = "cron"
+	ProcessType_Job    ProcessType = "job"
+	ProcessType_Web    ProcessType = "web"
+	ProcessType_Worker ProcessType = "worker"
+)
+
+type ProcessCron struct {
+	ID       string `yaml:"id"`
+	Schedule string `yaml:"schedule"`
+	Suffix   string `yaml:"suffix"`
+}
+
+type ProcessPortMap struct {
+	ContainerPort int32           `yaml:"container_port"`
+	HostPort      int32           `yaml:"host_port"`
+	Scheme        string          `yaml:"scheme"`
+	Protocol      PortmapProtocol `yaml:"protocol"`
+	Name          string          `yaml:"name"`
+}
+
+type PortmapProtocol string
+
+const (
+	PortmapProtocol_TCP PortmapProtocol = "TCP"
+	PortmapProtocol_UDP PortmapProtocol = "UDP"
+)
+
+type NameSorter []ProcessPortMap
+
+func (a NameSorter) Len() int           { return len(a) }
+func (a NameSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a NameSorter) Less(i, j int) bool { return a[i].Name < a[j].Name }
+
+type ProcessTls struct {
+	Enabled    bool   `yaml:"enabled"`
+	IssuerName string `yaml:"issuer_name"`
 }
 
 type ClusterIssuer struct {
@@ -303,459 +245,6 @@ type Job struct {
 	WorkingDir       string
 }
 
-func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
-	if input.Schedule == "" {
-		return batchv1.CronJob{}, fmt.Errorf("Schedule cannot be empty")
-	}
-	labels := map[string]string{
-		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-		"app.kubernetes.io/name":     input.ProcessType,
-		"app.kubernetes.io/part-of":  input.AppName,
-		"dokku.com/cron-id":          input.ID,
-	}
-	annotations := map[string]string{
-		"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
-		"dokku.com/builder-type":    input.ImageSourceType,
-		"dokku.com/cron-id":         input.ID,
-		"dokku.com/managed":         "true",
-	}
-
-	for key, value := range input.Labels {
-		labels[key] = value
-	}
-	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
-
-	env := []corev1.EnvVar{}
-	for key, value := range input.Env {
-		env = append(env, corev1.EnvVar{
-			Name:  key,
-			Value: value,
-		})
-	}
-
-	suffix := input.Suffix
-	if suffix == "" {
-		n := 5
-		b := make([]byte, n)
-		if _, err := rand.Read(b); err != nil {
-			panic(err)
-		}
-		suffix = strings.ToLower(fmt.Sprintf("%X", b))
-	}
-	annotations["dokku.com/job-suffix"] = suffix
-
-	podAnnotations := annotations
-	podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)
-
-	job := batchv1.CronJob{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
-			Namespace:   input.Namespace,
-			Labels:      labels,
-			Annotations: annotations,
-		},
-		Spec: batchv1.CronJobSpec{
-			ConcurrencyPolicy:          batchv1.AllowConcurrent,
-			FailedJobsHistoryLimit:     ptr.To(int32(10)),
-			Schedule:                   input.Schedule,
-			StartingDeadlineSeconds:    ptr.To(int64(60)),
-			SuccessfulJobsHistoryLimit: ptr.To(int32(10)),
-			Suspend:                    ptr.To(false),
-			TimeZone:                   ptr.To("Etc/UTC"),
-			JobTemplate: batchv1.JobTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels:      labels,
-					Annotations: annotations,
-				},
-				Spec: batchv1.JobSpec{
-					BackoffLimit:         ptr.To(int32(0)),
-					PodReplacementPolicy: ptr.To(batchv1.Failed),
-					Template: corev1.PodTemplateSpec{
-						ObjectMeta: metav1.ObjectMeta{
-							Labels:      labels,
-							Annotations: podAnnotations,
-						},
-						Spec: corev1.PodSpec{
-							Containers: []corev1.Container{
-								{
-									Args: input.Command,
-									Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-									Env:  env,
-									EnvFrom: []corev1.EnvFromSource{
-										{
-											SecretRef: &corev1.SecretEnvSource{
-												LocalObjectReference: corev1.LocalObjectReference{
-													Name: secretName,
-												},
-												Optional: ptr.To(true),
-											},
-										},
-									},
-									Image:           input.Image,
-									ImagePullPolicy: corev1.PullAlways,
-									Resources: corev1.ResourceRequirements{
-										Limits:   corev1.ResourceList{},
-										Requests: corev1.ResourceList{},
-									},
-									WorkingDir: input.WorkingDir,
-								},
-							},
-							RestartPolicy: corev1.RestartPolicyNever,
-						},
-					},
-				},
-			},
-		},
-	}
-
-	if input.Entrypoint != "" {
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command = []string{input.Entrypoint}
-	}
-
-	if input.RemoveContainer {
-		job.Spec.JobTemplate.Spec.TTLSecondsAfterFinished = ptr.To(int32(60))
-	}
-
-	if input.ImagePullSecrets != "" {
-		job.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
-			{
-				Name: input.ImagePullSecrets,
-			},
-		}
-	}
-
-	cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
-	if err != nil && cpuLimit != "" && cpuLimit != "0" {
-		cpuQuantity, err := resource.ParseQuantity(cpuLimit)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing cpu limit: %w", err)
-		}
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
-	} else {
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = resource.MustParse("500m")
-	}
-	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
-	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
-		nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
-		}
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
-	}
-	memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
-	if err != nil && memoryLimit != "" && memoryLimit != "0" {
-		memoryQuantity, err := resource.ParseQuantity(memoryLimit)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing memory limit: %w", err)
-		}
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
-	} else {
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = resource.MustParse("512Mi")
-	}
-
-	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
-	if err != nil && cpuRequest != "" && cpuRequest != "0" {
-		cpuQuantity, err := resource.ParseQuantity(cpuRequest)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing cpu request: %w", err)
-		}
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
-	} else {
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = resource.MustParse("500m")
-	}
-	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
-	if err != nil && memoryRequest != "" && memoryRequest != "0" {
-		memoryQuantity, err := resource.ParseQuantity(memoryRequest)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing memory request: %w", err)
-		}
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
-	} else {
-		job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = resource.MustParse("512Mi")
-	}
-
-	return job, nil
-}
-
-type Deployment struct {
-	AppName          string
-	Command          []string
-	Image            string
-	ImagePullSecrets string
-	ImageSourceType  string
-	Healthchecks     []appjson.Healthcheck
-	Namespace        string
-	PrimaryPort      int32
-	PortMaps         []PortMap
-	ProcessType      string
-	Replicas         int32
-	WorkingDir       string
-}
-
-func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
-	labels := map[string]string{
-		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-		"app.kubernetes.io/name":     input.ProcessType,
-		"app.kubernetes.io/part-of":  input.AppName,
-	}
-	annotations := map[string]string{
-		"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
-		"dokku.com/builder-type":    input.ImageSourceType,
-		"dokku.com/managed":         "true",
-	}
-	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
-
-	podAnnotations := annotations
-	podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)
-
-	deployment := appsv1.Deployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-			Namespace:   input.Namespace,
-			Labels:      labels,
-			Annotations: annotations,
-		},
-		Spec: appsv1.DeploymentSpec{
-			Replicas:             ptr.To(input.Replicas),
-			RevisionHistoryLimit: ptr.To(int32(5)),
-			Selector: &metav1.LabelSelector{
-				MatchLabels: labels,
-			},
-			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels:      labels,
-					Annotations: podAnnotations,
-				},
-				Spec: corev1.PodSpec{
-					Containers: []corev1.Container{
-						{
-							Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-							Env:  []corev1.EnvVar{},
-							EnvFrom: []corev1.EnvFromSource{
-								{
-									SecretRef: &corev1.SecretEnvSource{
-										LocalObjectReference: corev1.LocalObjectReference{
-											Name: secretName,
-										},
-										Optional: ptr.To(true),
-									},
-								},
-							},
-							Image:           input.Image,
-							ImagePullPolicy: corev1.PullAlways,
-							Resources: corev1.ResourceRequirements{
-								Limits:   corev1.ResourceList{},
-								Requests: corev1.ResourceList{},
-							},
-							WorkingDir: input.WorkingDir,
-						},
-					},
-				},
-			},
-		},
-	}
-
-	if len(input.Command) > 0 {
-		deployment.Spec.Template.Spec.Containers[0].Args = input.Command
-	}
-
-	if len(input.Healthchecks) > 0 {
-		livenessChecks := []corev1.Probe{}
-		readinessChecks := []corev1.Probe{}
-		startupChecks := []corev1.Probe{}
-		uptimeSeconds := []int32{}
-		for _, healthcheck := range input.Healthchecks {
-			probe := corev1.Probe{
-				ProbeHandler:        corev1.ProbeHandler{},
-				InitialDelaySeconds: healthcheck.InitialDelay,
-				PeriodSeconds:       healthcheck.Wait,
-				TimeoutSeconds:      healthcheck.Timeout,
-				FailureThreshold:    healthcheck.Attempts,
-				SuccessThreshold:    int32(1),
-			}
-			if len(healthcheck.Command) > 0 {
-				probe.ProbeHandler.Exec = &corev1.ExecAction{
-					Command: healthcheck.Command,
-				}
-			} else if healthcheck.Listening {
-				probe.ProbeHandler.TCPSocket = &corev1.TCPSocketAction{
-					Port: intstr.FromInt32(input.PrimaryPort),
-				}
-				for _, header := range healthcheck.HTTPHeaders {
-					if header.Name == "Host" {
-						probe.ProbeHandler.TCPSocket.Host = header.Value
-					}
-				}
-			} else if healthcheck.Path != "" {
-				probe.ProbeHandler.HTTPGet = &corev1.HTTPGetAction{
-					Path:        healthcheck.Path,
-					Port:        intstr.FromInt32(input.PrimaryPort),
-					HTTPHeaders: []corev1.HTTPHeader{},
-				}
-
-				if healthcheck.Scheme != "" {
-					probe.ProbeHandler.HTTPGet.Scheme = corev1.URIScheme(strings.ToUpper(healthcheck.Scheme))
-				}
-
-				for _, header := range healthcheck.HTTPHeaders {
-					probe.ProbeHandler.HTTPGet.HTTPHeaders = append(probe.ProbeHandler.HTTPGet.HTTPHeaders, corev1.HTTPHeader{
-						Name:  header.Name,
-						Value: header.Value,
-					})
-				}
-			} else if healthcheck.Uptime > 0 {
-				uptimeSeconds = append(uptimeSeconds, healthcheck.Uptime)
-			}
-
-			if healthcheck.Type == appjson.HealthcheckType_Liveness {
-				livenessChecks = append(livenessChecks, probe)
-			} else if healthcheck.Type == appjson.HealthcheckType_Readiness {
-				readinessChecks = append(readinessChecks, probe)
-			} else if healthcheck.Type == appjson.HealthcheckType_Startup {
-				startupChecks = append(startupChecks, probe)
-			}
-		}
-		if len(livenessChecks) > 1 {
-			common.LogWarn("Multiple liveness checks are not supported, only the first one will be used")
-		}
-		if len(readinessChecks) > 1 {
-			common.LogWarn("Multiple readiness checks are not supported, only the first one will be used")
-		}
-		if len(startupChecks) > 1 {
-			common.LogWarn("Multiple startup checks are not supported, only the first one will be used")
-		}
-		if len(uptimeSeconds) > 1 {
-			common.LogWarn("Multiple uptime checks are not supported, only the first one will be used")
-		}
-
-		if len(livenessChecks) > 0 {
-			deployment.Spec.Template.Spec.Containers[0].LivenessProbe = &livenessChecks[0]
-		}
-		if len(readinessChecks) > 0 {
-			deployment.Spec.Template.Spec.Containers[0].ReadinessProbe = &readinessChecks[0]
-		}
-		if len(startupChecks) > 0 {
-			deployment.Spec.Template.Spec.Containers[0].StartupProbe = &startupChecks[0]
-		}
-		if len(uptimeSeconds) > 0 {
-			deployment.Spec.MinReadySeconds = uptimeSeconds[0]
-		}
-	}
-
-	if input.ProcessType == "web" {
-		for _, portMap := range input.PortMaps {
-			protocol := "TCP"
-			if portMap.Scheme == "udp" {
-				protocol = "UDP"
-			}
-			deployment.Spec.Template.Spec.Containers[0].Ports = append(deployment.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{
-				Name:          portMap.String(),
-				ContainerPort: portMap.ContainerPort,
-				Protocol:      corev1.Protocol(protocol),
-			})
-		}
-
-		deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
-			Name:  "PORT",
-			Value: fmt.Sprint(input.PrimaryPort),
-		})
-	}
-
-	if input.ImagePullSecrets != "" {
-		deployment.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
-			{
-				Name: input.ImagePullSecrets,
-			},
-		}
-	}
-
-	cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
-	if err != nil && cpuLimit != "" && cpuLimit != "0" {
-		cpuQuantity, err := resource.ParseQuantity(cpuLimit)
-		if err != nil {
-			return deployment, fmt.Errorf("Error parsing cpu limit: %w", err)
-		}
-		deployment.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
-	}
-	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
-	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
-		nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
-		if err != nil {
-			return deployment, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
-		}
-		deployment.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
-	}
-	memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
-	if err != nil && memoryLimit != "" && memoryLimit != "0" {
-		memoryQuantity, err := resource.ParseQuantity(memoryLimit)
-		if err != nil {
-			return deployment, fmt.Errorf("Error parsing memory limit: %w", err)
-		}
-		deployment.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
-	}
-
-	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
-	if err != nil && cpuRequest != "" && cpuRequest != "0" {
-		cpuQuantity, err := resource.ParseQuantity(cpuRequest)
-		if err != nil {
-			return deployment, fmt.Errorf("Error parsing cpu request: %w", err)
-		}
-		deployment.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
-	}
-	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
-	if err != nil && memoryRequest != "" && memoryRequest != "0" {
-		memoryQuantity, err := resource.ParseQuantity(memoryRequest)
-		if err != nil {
-			return deployment, fmt.Errorf("Error parsing memory request: %w", err)
-		}
-		deployment.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
-	}
-
-	return deployment, nil
-}
-
-type IngressRouteEntrypoint string
-
-const (
-	IngressRouteEntrypoint_HTTP  IngressRouteEntrypoint = "web"
-	IngressRouteEntrypoint_HTTPS IngressRouteEntrypoint = "websecure"
-)
-
-type IngressRoute struct {
-	AppName     string
-	Entrypoints []IngressRouteEntrypoint
-	Namespace   string
-	PortMap     PortMap
-	ProcessType string
-	ServiceName string
-}
-
-func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressRoute {
-	labels := map[string]string{
-		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
-		"app.kubernetes.io/name":     input.ProcessType,
-		"app.kubernetes.io/part-of":  input.AppName,
-	}
-	annotations := map[string]string{
-		"dokku.com/managed": "true",
-	}
-
-	port := input.PortMap.String()
-	ingressRoute := traefikv1alpha1.IngressRoute{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        fmt.Sprintf("%s-%s", input.ServiceName, port),
-			Namespace:   input.Namespace,
-			Labels:      labels,
-			Annotations: annotations,
-		},
-		Spec: traefikv1alpha1.IngressRouteSpec{
-			EntryPoints: []string{string(IngressRouteEntrypoint_HTTP)},
-		},
-	}
-
-	return ingressRoute
-}
-
 func templateKubernetesJob(input Job) (batchv1.Job, error) {
 	labels := map[string]string{
 		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
@@ -907,137 +396,6 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 	return job, nil
 }
 
-type Secret struct {
-	AppName   string
-	Env       map[string]string
-	Namespace string
-}
-
-func templateKubernetesSecret(input Secret) corev1.Secret {
-	secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
-	labels := map[string]string{
-		"app.kubernetes.io/instance": secretName,
-		"app.kubernetes.io/name":     fmt.Sprintf("%s-env", input.AppName),
-		"app.kubernetes.io/part-of":  input.AppName,
-	}
-
-	annotations := map[string]string{
-		"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
-		"dokku.com/managed":         "true",
-	}
-	secret := corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        secretName,
-			Namespace:   input.Namespace,
-			Labels:      labels,
-			Annotations: annotations,
-		},
-		Data: map[string][]byte{},
-	}
-
-	return secret
-}
-
-type Service struct {
-	AppName   string
-	Namespace string
-	PortMaps  []PortMap
-}
-
-func templateKubernetesService(input Service) corev1.Service {
-	labels := map[string]string{
-		"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, "web"),
-		"app.kubernetes.io/name":     "web",
-		"app.kubernetes.io/part-of":  input.AppName,
-	}
-	annotations := map[string]string{
-		"dokku.com/managed": "true",
-	}
-	service := corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        fmt.Sprintf("%s-%s", input.AppName, "web"),
-			Namespace:   input.Namespace,
-			Labels:      labels,
-			Annotations: annotations,
-		},
-		Spec: corev1.ServiceSpec{
-			Selector: labels,
-		},
-	}
-
-	for _, portMap := range input.PortMaps {
-		protocol := "TCP"
-		if portMap.Scheme == "udp" {
-			protocol = "UDP"
-		}
-		service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{
-			Name:       portMap.String(),
-			Port:       portMap.HostPort,
-			TargetPort: intstr.FromString(portMap.String()),
-			Protocol:   corev1.Protocol(protocol),
-		})
-	}
-
-	return service
-}
-
-type WriteResourceInput struct {
-	AppendContents string
-	Object         runtime.Object
-	Path           string
-	Replacements   *orderedmap.OrderedMap[string, string]
-}
-
-func writeResourceToFile(input WriteResourceInput) error {
-	common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
-	printr := printers.NewTypeSetter(runtimeScheme).ToPrinter(&printers.YAMLPrinter{})
-	handle, err := os.Create(input.Path)
-	if err != nil {
-		return fmt.Errorf("Error creating template file: %w", err)
-	}
-	defer handle.Close()
-
-	if err := printr.PrintObj(input.Object, handle); err != nil {
-		return fmt.Errorf("Error writing template file: %w", err)
-	}
-
-	if input.Replacements != nil {
-		b, err := os.ReadFile(input.Path)
-		if err != nil {
-			return fmt.Errorf("Error reading template file: %w", err)
-		}
-
-		contents := string(b)
-		for pair := input.Replacements.Oldest(); pair != nil; pair = pair.Next() {
-			contents = strings.ReplaceAll(string(contents), pair.Key, pair.Value)
-		}
-
-		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
-		if err != nil {
-			return fmt.Errorf("Error updating template file with replacements: %w", err)
-		}
-	}
-
-	if input.AppendContents != "" {
-		b, err := os.ReadFile(input.Path)
-		if err != nil {
-			return fmt.Errorf("Error reading template file: %w", err)
-		}
-
-		contents := string(b) + "\n" + input.AppendContents
-
-		err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
-		if err != nil {
-			return fmt.Errorf("Error updating template file with replacements: %w", err)
-		}
-	}
-
-	if os.Getenv("DOKKU_TRACE") == "1" {
-		common.CatFile(input.Path)
-	}
-	return nil
-}
-
 type WriteYamlInput struct {
 	Object interface{}
 	Path   string
diff --git a/plugins/scheduler-k3s/templates/certificate-append.yaml b/plugins/scheduler-k3s/templates/certificate-append.yaml
deleted file mode 100644
index 538ddfdaa..000000000
--- a/plugins/scheduler-k3s/templates/certificate-append.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-  dnsNames:
-    {{- range .Values.processes.PROCESS_TYPE.domains }}
-    - {{ . }}
-    {{- end }}
\ No newline at end of file
diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml
new file mode 100644
index 000000000..30c385042
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml
@@ -0,0 +1,32 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := .Values.processes.PROCESS_NAME }}
+{{- if and $config.web.tls $config.domains }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: {{ $.Values.global.app_name }}-{{ $processName }}
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  issuerRef:
+    kind: ClusterIssuer
+    name: {{ $config.tls.issuer_name }}
+  secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
+  secretTemplate:
+    annotations:
+      dokku.com/managed: "true"
+    labels:
+      app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }}
+      app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+status: {}
+  dnsNames:
+    {{- range $config.domains }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
new file mode 100644
index 000000000..13fbf4109
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
@@ -0,0 +1,103 @@
+{{- $processName := "CRON_ID" }}
+{{- $config := .Values.processes.CRON_ID }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  annotations:
+    app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
+    dokku.com/builder-type: {{ $.Values.global.image.type }}
+    dokku.com/cron-id: {{ $config.cron.id }}
+    dokku.com/job-suffix: {{ $config.cron.suffix }}
+    dokku.com/managed: "true"
+    kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
+    app.kubernetes.io/name: cron
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+    dokku.com/cron-id: {{ $config.cron.id }}
+  name: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
+  namespace: default
+spec:
+  concurrencyPolicy: Allow
+  failedJobsHistoryLimit: 10
+  jobTemplate:
+    metadata:
+      annotations:
+        app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
+        dokku.com/builder-type: {{ $.Values.global.image.type }}
+        dokku.com/cron-id: {{ $config.cron.id }}
+        dokku.com/job-suffix: {{ $config.cron.suffix }}
+        dokku.com/managed: "true"
+        kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
+        app.kubernetes.io/name: cron
+        app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+        dokku.com/cron-id: {{ $config.cron.id }}
+    spec:
+      backoffLimit: 0
+      podReplacementPolicy: Failed
+      template:
+        metadata:
+          annotations:
+            app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
+            dokku.com/builder-type: {{ $.Values.global.image.type }}
+            dokku.com/cron-id: {{ $config.cron.id }}
+            dokku.com/job-suffix: {{ $config.cron.suffix }}
+            dokku.com/managed: "true"
+            kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
+          creationTimestamp: null
+          labels:
+            app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
+            app.kubernetes.io/name: cron
+            app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+            dokku.com/cron-id: {{ $config.cron.id }}
+        spec:
+          containers:
+          - args:
+            {{- range $config.args }}
+            - {{ . }}
+            {{- end }}
+            envFrom:
+            - secretRef:
+                name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
+                optional: true
+            image: {{ $.Values.global.image.name }}
+            imagePullPolicy: Always
+            name: {{ $.Values.global.app_name }}-cron
+            {{- if and $config.resources (or $config.resources.limits $config.resources.requests) }}
+            resources:
+              {{- if $config.resources.limits }}
+              limits:
+                {{- if $config.resources.limits.cpu }}
+                cpu: {{ $config.resources.limits.cpu }}
+                {{- end }}
+                {{- if $config.resources.limits.memory }}
+                memory: {{ $config.resources.limits.memory }}
+                {{- end }}
+              {{- end }}
+              {{- if $config.resources.requests }}
+              requests:
+                {{- if $config.resources.requests.cpu }}
+                cpu: {{ $config.resources.requests.cpu }}
+                {{- end }}
+                {{- if $config.resources.requests.memory }}
+                memory: {{ $config.resources.requests.memory }}
+                {{- end }}
+              {{- end }}
+            {{- end }}
+            {{- if $.Values.global.image.working_dir }}
+            workingDir: {{ $.Values.global.image.working_dir }}
+            {{- end }}
+          {{- if $.Values.global.image.image_pull_secrets }}
+          imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }}
+          {{- end }}
+          restartPolicy: Never
+          ttlSecondsAfterFinished: 60
+  schedule: {{ $config.cron.schedule }}
+  startingDeadlineSeconds: 60
+  successfulJobsHistoryLimit: 10
+  suspend: false
+  timeZone: Etc/UTC
\ No newline at end of file
diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml
new file mode 100644
index 000000000..89c6fbb7e
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml
@@ -0,0 +1,96 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := .Values.processes.PROCESS_NAME }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
+    dokku.com/builder-type: {{ $.Values.global.image.type }}
+    dokku.com/managed: "true"
+    kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: {{ $.Values.global.app_name }}-{{ $processName }}
+  namespace: default
+spec:
+  replicas: {{ $config.replicas }}
+  revisionHistoryLimit: 5
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+      app.kubernetes.io/name: {{ $processName }}
+      app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  strategy: {}
+  template:
+    metadata:
+      annotations:
+        app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
+        dokku.com/builder-type: {{ $.Values.global.image.type }}
+        dokku.com/managed: "true"
+        kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+        app.kubernetes.io/name: {{ $processName }}
+        app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+    spec:
+      containers:
+      - args:
+        {{- range $config.args }}
+        - {{ . }}
+        {{- end }}
+        {{- if eq $processName "web" }}
+        env:
+        - name: PORT
+          value: "{{ $.Values.global.primary_port }}"
+        {{- end }}
+        envFrom:
+        - secretRef:
+            name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
+            optional: true
+        image: {{ $.Values.global.image.name }}
+        imagePullPolicy: Always
+        name: {{ $.Values.global.app_name }}-{{ $processName }}
+        {{- if eq $processName "web" }}
+        ports:
+        {{- range $pdx, $port_map := $config.web.port_maps }}
+        - containerPort: {{ $port_map.container_port }}
+          name: {{ $port_map.name }}
+          protocol: {{ $port_map.protocol }}
+        {{- end }}
+        {{- end }}
+        {{- if and $config.resources (or $config.resources.limits $config.resources.requests) }}
+        resources:
+          {{- if $config.resources.limits }}
+          limits:
+            {{- if $config.resources.limits.cpu }}
+            cpu: {{ $config.resources.limits.cpu }}
+            {{- end }}
+            {{- if $config.resources.limits.memory }}
+            memory: {{ $config.resources.limits.memory }}
+            {{- end }}
+          {{- end }}
+          {{- if $config.resources.requests }}
+          requests:
+            {{- if $config.resources.requests.cpu }}
+            cpu: {{ $config.resources.requests.cpu }}
+            {{- end }}
+            {{- if $config.resources.requests.memory }}
+            memory: {{ $config.resources.requests.memory }}
+            {{- end }}
+          {{- end }}
+        {{- end }}
+        {{- if $config.healthchecks.startup }}
+        startupProbe:
+          {{ $config.healthchecks.startup | toJson | indent 10 }}
+        {{- end }}
+        {{- if $.Values.global.image.working_dir }}
+        workingDir: {{ $.Values.global.image.working_dir }}
+        {{- end }}
+      {{- if $.Values.global.image.image_pull_secrets }}
+      imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }}
+      {{- end }}
diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
new file mode 100644
index 000000000..444a25c88
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
@@ -0,0 +1,35 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := .Values.processes.PROCESS_NAME }}
+{{- if $config.web.domains }}
+{{- range $pdx, $port_map := $config.web.port_maps }}
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }}
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  entryPoints:
+  - web
+  {{- if and $config.web.tls (eq $port_map "https") }}
+  - websecure
+  {{- end }}
+  routes:
+    {{- range $ddx, $domain := $config.web.domains }}
+    - kind: Rule
+      match: Host(`{{ $domain }}`)
+      services:
+      - name: {{ $.Values.global.app_name }}-{{ $processName }}
+        namespace: {{ $.Values.global.namespace }}
+        passHostHeader: true
+        port: {{ $port_map.name }}
+        scheme: {{ $port_map.scheme }}
+    {{- end }}
+{{- end }}
+{{- end }}
diff --git a/plugins/scheduler-k3s/templates/chart/secret.yaml b/plugins/scheduler-k3s/templates/chart/secret.yaml
new file mode 100644
index 000000000..a0115ac2b
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/secret.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  annotations:
+    app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
+    dokku.com/managed: "true"
+  labels:
+    app.kubernetes.io/instance: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
+    app.kubernetes.io/name: env-{{ $.Values.global.app_name }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
+  namespace: {{ $.Values.global.namespace }}
+{{- with .Values.global.secrets }}
+data:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
\ No newline at end of file
diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml
new file mode 100644
index 000000000..a3ed18c83
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/service.yaml
@@ -0,0 +1,26 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := .Values.processes.PROCESS_NAME }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: {{ $.Values.global.app_name }}-{{ $processName }}
+  namespace: default
+spec:
+  ports:
+  {{- range $pdx, $port_map := $config.web.port_maps }}
+  - name: {{ $port_map.name }}
+    port: {{ $port_map.host_port }}
+    protocol: TCP
+    targetPort: {{ $port_map.name }}
+  {{- end }}
+  selector:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
diff --git a/plugins/scheduler-k3s/templates/cert-manager.yaml b/plugins/scheduler-k3s/templates/helm-config/cert-manager.yaml
similarity index 100%
rename from plugins/scheduler-k3s/templates/cert-manager.yaml
rename to plugins/scheduler-k3s/templates/helm-config/cert-manager.yaml
diff --git a/plugins/scheduler-k3s/templates/longhorn.yaml b/plugins/scheduler-k3s/templates/helm-config/longhorn.yaml
similarity index 100%
rename from plugins/scheduler-k3s/templates/longhorn.yaml
rename to plugins/scheduler-k3s/templates/helm-config/longhorn.yaml
diff --git a/plugins/scheduler-k3s/templates/traefik-config.yaml b/plugins/scheduler-k3s/templates/helm-config/traefik-config.yaml
similarity index 100%
rename from plugins/scheduler-k3s/templates/traefik-config.yaml
rename to plugins/scheduler-k3s/templates/helm-config/traefik-config.yaml
diff --git a/plugins/scheduler-k3s/templates/ingress-routes-append.yaml b/plugins/scheduler-k3s/templates/ingress-routes-append.yaml
deleted file mode 100644
index 237ebeef9..000000000
--- a/plugins/scheduler-k3s/templates/ingress-routes-append.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-  {{ if .Values.processes.PROCESS_TYPE.tls }}
-  - websecure
-  {{- end }}
-  routes:
-    {{- range .Values.processes.PROCESS_TYPE.domains }}
-    - kind: Rule
-      match: Host(`{{ . }}`)
-      services:
-      - name: APP_NAME-PROCESS_TYPE
-        namespace: NAMESPACE
-        passHostHeader: true
-        port: PORT_MAPPING
-        scheme: PORT_SCHEME
-    {{- end }}
-  {{ if .Values.processes.PROCESS_TYPE.tls }}
-  tls:
-    secretName: tls-APP_NAME-PROCESS_TYPE
-  {{- end }}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index fea7c18ac..e5ec1492c 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -3,6 +3,7 @@ package scheduler_k3s
 import (
 	"bufio"
 	"context"
+	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -26,9 +27,7 @@ import (
 	"github.com/kballard/go-shellquote"
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/ryanuber/columnize"
-	orderedmap "github.com/wk8/go-ordered-map/v2"
 	"gopkg.in/yaml.v3"
-	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/kubernetes/pkg/client/conditions"
@@ -227,34 +226,26 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	if err != nil {
 		return fmt.Errorf("Error creating chart directory: %w", err)
 	}
-	defer os.RemoveAll(chartDir)
+	// defer os.RemoveAll(chartDir)
 
 	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
 		return fmt.Errorf("Error creating chart templates directory: %w", err)
 	}
 
 	deploymentId := time.Now().Unix()
-	replacements := orderedmap.New[string, string]()
-	replacements.Set("DEPLOYMENT_ID_QUOTED", "{{.Values.deploment_id | quote}}")
-	replacements.Set("DEPLOYMENT_ID", "{{.Values.deploment_id}}")
-
-	secret := templateKubernetesSecret(Secret{
-		AppName:   appName,
-		Env:       env.Map(),
-		Namespace: namespace,
-	})
-	err = writeResourceToFile(WriteResourceInput{
-		Object:       &secret,
-		Path:         filepath.Join(chartDir, "templates/secret.yaml"),
-		Replacements: replacements,
-		AppendContents: `{{- with .Values.secrets }}
-data:
-  {{- toYaml . | nindent 2 }}
-{{- end }}
-`,
-	})
+	b, err := templates.ReadFile("templates/chart/secret.yaml")
 	if err != nil {
-		return fmt.Errorf("Error printing deployment: %w", err)
+		return fmt.Errorf("Error reading secret template: %w", err)
+	}
+
+	filename := filepath.Join(chartDir, "templates", "secrets.yaml")
+	err = os.WriteFile(filename, b, os.FileMode(0644))
+	if err != nil {
+		return fmt.Errorf("Error writing secrets template: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(filename)
 	}
 
 	portMaps, err := getPortMaps(appName)
@@ -276,125 +267,12 @@ data:
 	}
 
 	workingDir := common.GetWorkingDir(appName, image)
-	deployments := map[string]appsv1.Deployment{}
-	i := 0
-	for processType := range processes {
-		startCommand, err := getStartCommand(StartCommandInput{
-			AppName:         appName,
-			ProcessType:     processType,
-			ImageSourceType: imageSourceType,
-			Port:            primaryPort,
-			Env:             env.Map(),
-		})
-		if err != nil {
-			return fmt.Errorf("Error getting start command for deployment: %w", err)
-		}
-
-		i++
-		replicaCountPlaceholder := int32(i * 1000)
-
-		healthchecks, ok := appJSON.Healthchecks[processType]
-		if !ok {
-			healthchecks = []appjson.Healthcheck{}
-		}
-
-		// todo: implement deployment annotations
-		// todo: implement pod annotations
-		// todo: implement volumes
-		deployment, err := templateKubernetesDeployment(Deployment{
-			AppName:          appName,
-			Command:          startCommand.Command,
-			Image:            image,
-			ImagePullSecrets: imagePullSecrets,
-			ImageSourceType:  imageSourceType,
-			Healthchecks:     healthchecks,
-			Namespace:        namespace,
-			PrimaryPort:      primaryPort,
-			PortMaps:         portMaps,
-			ProcessType:      processType,
-			Replicas:         replicaCountPlaceholder,
-			WorkingDir:       workingDir,
-		})
-		if err != nil {
-			return fmt.Errorf("Error templating deployment: %w", err)
-		}
-
-		replacements.Set(fmt.Sprintf("replicas: %d", replicaCountPlaceholder), fmt.Sprintf("replicas: {{.Values.processes.%s.replicas}}", processType))
-		deployments[processType] = deployment
-		err = writeResourceToFile(WriteResourceInput{
-			Object:       &deployment,
-			Path:         filepath.Join(chartDir, fmt.Sprintf("templates/deployment-%s.yaml", deployment.Name)),
-			Replacements: replacements,
-		})
-		if err != nil {
-			return fmt.Errorf("Error printing deployment: %w", err)
-		}
-
-		replacements.Delete(fmt.Sprintf("replicas: %d", replicaCountPlaceholder))
-	}
 
 	cronEntries, err := cron.FetchCronEntries(appName)
 	if err != nil {
 		return fmt.Errorf("Error fetching cron entries: %w", err)
 	}
 
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Error creating kubernetes client: %w", err)
-	}
-
-	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
-		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
-		Namespace:     namespace,
-	})
-	if err != nil {
-		return fmt.Errorf("Error listing cron jobs: %w", err)
-	}
-
-	for _, cronEntry := range cronEntries {
-		suffix := ""
-		for _, cronJob := range cronJobs {
-			if cronJob.Labels["dokku.com/cron-id"] == cronEntry.ID {
-				var ok bool
-				suffix, ok = cronJob.Annotations["dokku.com/job-suffix"]
-				if !ok {
-					suffix = ""
-				}
-			}
-		}
-
-		words, err := shellquote.Split(cronEntry.Command)
-		if err != nil {
-			return fmt.Errorf("Error parsing cron command: %w", err)
-		}
-		cronJob, err := templateKubernetesCronJob(Job{
-			AppName:          appName,
-			Command:          words,
-			Env:              map[string]string{},
-			ID:               cronEntry.ID,
-			Image:            image,
-			ImagePullSecrets: imagePullSecrets,
-			ImageSourceType:  imageSourceType,
-			Namespace:        namespace,
-			ProcessType:      "cron",
-			Schedule:         cronEntry.Schedule,
-			Suffix:           suffix,
-			WorkingDir:       workingDir,
-		})
-		if err != nil {
-			return fmt.Errorf("Error templating cron job: %w", err)
-		}
-
-		err = writeResourceToFile(WriteResourceInput{
-			Object:       &cronJob,
-			Path:         filepath.Join(chartDir, fmt.Sprintf("templates/cron-job-%s.yaml", cronEntry.ID)),
-			Replacements: replacements,
-		})
-		if err != nil {
-			return fmt.Errorf("Error printing cron job: %w", err)
-		}
-	}
-
 	issuerName := "letsencrypt-stag"
 	server := getComputedLetsencryptServer(appName)
 	if server == "prod" || server == "production" {
@@ -412,22 +290,7 @@ data:
 	}
 
 	domains := []string{}
-	if deployment, ok := deployments["web"]; ok {
-		service := templateKubernetesService(Service{
-			AppName:   appName,
-			Namespace: namespace,
-			PortMaps:  portMaps,
-		})
-
-		err := writeResourceToFile(WriteResourceInput{
-			Object:       &service,
-			Path:         filepath.Join(chartDir, "templates/service-web.yaml"),
-			Replacements: replacements,
-		})
-		if err != nil {
-			return fmt.Errorf("Error printing service: %w", err)
-		}
-
+	if _, ok := processes["web"]; ok {
 		err = common.PlugnTrigger("domains-vhost-enabled", []string{appName}...)
 		if err == nil {
 			b, err := common.PlugnTriggerOutput("domains-list", []string{appName}...)
@@ -442,34 +305,6 @@ data:
 				}
 			}
 		}
-
-		err = createIngressRoutesFiles(CreateIngressRoutesInput{
-			AppName:     appName,
-			ChartDir:    chartDir,
-			Deployment:  deployment,
-			Namespace:   namespace,
-			PortMaps:    portMaps,
-			ProcessType: "web",
-			Service:     service,
-		})
-		if err != nil {
-			return fmt.Errorf("Error creating ingress routes: %w", err)
-		}
-
-		err = createCertificateFile(CreateCertificateFileInput{
-			ChartDir: chartDir,
-			Certificate: Certificate{
-				AppName:   appName,
-				Name:      fmt.Sprintf("%s-%s", appName, "web"),
-				Namespace: namespace,
-				TLS:       tls,
-			},
-			IssuerName:  issuerName,
-			ProcessType: "web",
-		})
-		if err != nil {
-			return fmt.Errorf("Error creating certificate files: %w", err)
-		}
 	}
 
 	chart := &Chart{
@@ -488,25 +323,181 @@ data:
 	}
 
 	values := &Values{
-		DeploymentID: fmt.Sprint(deploymentId),
-		Secrets:      map[string]string{},
-		Processes:    map[string]ProcessValues{},
+		Global: GlobalValues{
+			AppName:      appName,
+			DeploymentID: fmt.Sprint(deploymentId),
+			Image: GlobalImage{
+				ImagePullSecrets: imagePullSecrets,
+				Name:             image,
+				Type:             imageSourceType,
+				WorkingDir:       workingDir,
+			},
+			Namespace:   namespace,
+			PrimaryPort: primaryPort,
+			Secrets:     map[string]string{},
+		},
+		Processes: map[string]ProcessValues{},
 	}
+
 	for processType, processCount := range processes {
+		// todo: implement deployment annotations
+		// todo: implement pod annotations
+		// todo: implement volumes
+
+		healthchecks, ok := appJSON.Healthchecks[processType]
+		if !ok {
+			healthchecks = []appjson.Healthcheck{}
+		}
+		processHealthchecks := getProcessHealtchecks(healthchecks, primaryPort)
+
+		startCommand, err := getStartCommand(StartCommandInput{
+			AppName:         appName,
+			ProcessType:     processType,
+			ImageSourceType: imageSourceType,
+			Port:            primaryPort,
+			Env:             env.Map(),
+		})
+		if err != nil {
+			return fmt.Errorf("Error getting start command for deployment: %w", err)
+		}
+		args := startCommand.Command
+
+		processResources, err := getProcessResources(appName, processType)
+		if err != nil {
+			return fmt.Errorf("Error getting process resources: %w", err)
+		}
+
 		processValues := ProcessValues{
-			Replicas: int32(processCount),
+			Args:         args,
+			Healthchecks: processHealthchecks,
+			ProcessType:  ProcessType_Worker,
+			Replicas:     int32(processCount),
+			Resources:    processResources,
 		}
 		if processType == "web" {
-			sort.Strings(domains)
-			processValues.Domains = domains
-			processValues.TLS = tls
+			processValues.Web = ProcessWeb{
+				Domains:  domains,
+				PortMaps: []ProcessPortMap{},
+				TLS:      tls,
+			}
+			processValues.ProcessType = ProcessType_Web
+			for _, portMap := range portMaps {
+				protocol := PortmapProtocol_TCP
+				if portMap.Scheme == "udp" {
+					protocol = PortmapProtocol_UDP
+				}
+				processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
+					ContainerPort: portMap.ContainerPort,
+					HostPort:      portMap.HostPort,
+					Name:          portMap.String(),
+					Protocol:      protocol,
+					Scheme:        portMap.Scheme,
+				})
+			}
+
+			sort.Sort(NameSorter(processValues.Web.PortMaps))
+			sort.Strings(processValues.Web.Domains)
 		}
 
 		values.Processes[processType] = processValues
+
+		for _, templateName := range []string{"deployment", "service", "certificate", "ingress-route"} {
+			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
+			if err != nil {
+				return fmt.Errorf("Error reading %s template: %w", templateName, err)
+			}
+
+			filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s-%s.yaml", templateName, processType))
+			contents := strings.ReplaceAll(string(b), "PROCESS_NAME", processType)
+			err = os.WriteFile(filename, []byte(contents), os.FileMode(0644))
+			if err != nil {
+				return fmt.Errorf("Error writing %s template: %w", templateName, err)
+			}
+
+			if os.Getenv("DOKKU_TRACE") == "1" {
+				common.CatFile(filename)
+			}
+		}
+
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
+		Namespace:     namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing cron jobs: %w", err)
+	}
+	for _, cronEntry := range cronEntries {
+		// todo: implement deployment annotations
+		// todo: implement pod annotations
+		// todo: implement volumes
+		suffix := ""
+		for _, cronJob := range cronJobs {
+			if cronJob.Labels["dokku.com/cron-id"] == cronEntry.ID {
+				var ok bool
+				suffix, ok = cronJob.Annotations["dokku.com/job-suffix"]
+				if !ok {
+					suffix = ""
+				}
+			}
+		}
+		if suffix == "" {
+			n := 5
+			b := make([]byte, n)
+			if _, err := rand.Read(b); err != nil {
+				panic(err)
+			}
+			suffix = strings.ToLower(fmt.Sprintf("%X", b))
+		}
+
+		words, err := shellquote.Split(cronEntry.Command)
+		if err != nil {
+			return fmt.Errorf("Error parsing cron command: %w", err)
+		}
+
+		processResources, err := getProcessResources(appName, cronEntry.ID)
+		if err != nil {
+			return fmt.Errorf("Error getting process resources: %w", err)
+		}
+
+		processValues := ProcessValues{
+			Args: words,
+			Cron: ProcessCron{
+				ID:       cronEntry.ID,
+				Schedule: cronEntry.Schedule,
+				Suffix:   suffix,
+			},
+			ProcessType: ProcessType_Cron,
+			Replicas:    1,
+			Resources:   processResources,
+		}
+		values.Processes[cronEntry.ID] = processValues
+
+		b, err := templates.ReadFile("templates/chart/cron-job.yaml")
+		if err != nil {
+			return fmt.Errorf("Error reading cron job template: %w", err)
+		}
+
+		cronFile := filepath.Join(chartDir, "templates", fmt.Sprintf("cron-job-%s.yaml", cronEntry.ID))
+		contents := strings.ReplaceAll(string(b), "CRON_ID", cronEntry.ID)
+		err = os.WriteFile(cronFile, []byte(contents), os.FileMode(0644))
+		if err != nil {
+			return fmt.Errorf("Error writing cron job template: %w", err)
+		}
+
+		if os.Getenv("DOKKU_TRACE") == "1" {
+			common.CatFile(cronFile)
+		}
 	}
 
 	for key, value := range env.Map() {
-		values.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value))
+		values.Global.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value))
 	}
 
 	err = writeYaml(WriteYamlInput{

From 1e559f20e5d8d3b11c783e4e63213b393192ccb0 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 01:59:00 -0500
Subject: [PATCH 091/113] refactor: use helper function to get process
 resources

---
 plugins/scheduler-k3s/template.go | 49 ++++++++++---------------------
 1 file changed, 16 insertions(+), 33 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 845ebe3e7..03cc63ade 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -351,45 +351,28 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 		}
 	}
 
-	cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
-	if err != nil && cpuLimit != "" && cpuLimit != "0" {
-		cpuQuantity, err := resource.ParseQuantity(cpuLimit)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing cpu limit: %w", err)
-		}
+	processResources, err := getProcessResources(input.AppName, input.ProcessType)
+	if err != nil {
+		return job, fmt.Errorf("Error getting process resources: %w", err)
+	}
+	if processResources.Limits.CPU != "" {
+		cpuQuantity, _ := resource.ParseQuantity(processResources.Limits.CPU)
 		job.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
 	}
-	nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
-	if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
-		nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
-		}
-		job.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
-	}
-	memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
-	if err != nil && memoryLimit != "" && memoryLimit != "0" {
-		memoryQuantity, err := resource.ParseQuantity(memoryLimit)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing memory limit: %w", err)
-		}
+	if processResources.Limits.Memory != "" {
+		memoryQuantity, _ := resource.ParseQuantity(processResources.Limits.Memory)
 		job.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
 	}
-
-	cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
-	if err != nil && cpuRequest != "" && cpuRequest != "0" {
-		cpuQuantity, err := resource.ParseQuantity(cpuRequest)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing cpu request: %w", err)
-		}
+	if processResources.Limits.NvidiaGPU != "" {
+		nvidiaGpuQuantity, _ := resource.ParseQuantity(processResources.Limits.NvidiaGPU)
+		job.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
+	}
+	if processResources.Requests.CPU != "" {
+		cpuQuantity, _ := resource.ParseQuantity(processResources.Requests.CPU)
 		job.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
 	}
-	memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
-	if err != nil && memoryRequest != "" && memoryRequest != "0" {
-		memoryQuantity, err := resource.ParseQuantity(memoryRequest)
-		if err != nil {
-			return job, fmt.Errorf("Error parsing memory request: %w", err)
-		}
+	if processResources.Requests.Memory != "" {
+		memoryQuantity, _ := resource.ParseQuantity(processResources.Requests.Memory)
 		job.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
 	}
 

From 56f25c4855d456ec44d17c31a412de2cc2d8fbc3 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 03:21:05 -0500
Subject: [PATCH 092/113] fix: ensure the known_hosts file exists before
 attempting to ssh onto a server

---
 plugins/common/ssh.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/plugins/common/ssh.go b/plugins/common/ssh.go
index 4d0c08e7f..0c405d448 100644
--- a/plugins/common/ssh.go
+++ b/plugins/common/ssh.go
@@ -254,6 +254,10 @@ func (task SshTask) Execute(ctx context.Context) (SshResult, error) {
 		return SshResult{}, fmt.Errorf("failed to load known hosts: %w", err)
 	}
 
+	if err := TouchFile(filepath.Join(os.Getenv("DOKKU_ROOT"), ".ssh", "known_hosts")); err != nil {
+		return SshResult{}, fmt.Errorf("failed to touch known_hosts file: %w", err)
+	}
+
 	if task.AllowUknownHosts {
 		callback = ssh.InsecureIgnoreHostKey()
 	}

From c6838b645e944f4a4276aaa7e1b1ea82d78c5316 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 03:21:21 -0500
Subject: [PATCH 093/113] chore: update go mod

---
 plugins/scheduler-k3s/go.mod | 5 +----
 plugins/scheduler-k3s/go.sum | 6 ------
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod
index 32de4e5dc..2348a7fa1 100644
--- a/plugins/scheduler-k3s/go.mod
+++ b/plugins/scheduler-k3s/go.mod
@@ -18,12 +18,10 @@ require (
 	github.com/ryanuber/columnize v2.1.2+incompatible
 	github.com/spf13/pflag v1.0.5
 	github.com/traefik/traefik/v2 v2.10.7
-	github.com/wk8/go-ordered-map/v2 v2.1.8
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.13.3
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0
-	k8s.io/cli-runtime v0.29.0
 	k8s.io/client-go v0.29.0
 	k8s.io/kubectl v0.28.4
 	k8s.io/kubernetes v1.29.1
@@ -43,9 +41,7 @@ require (
 	github.com/Microsoft/hcsshim v0.11.0 // indirect
 	github.com/alexellis/go-execute/v2 v2.2.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
-	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
@@ -167,6 +163,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
+	k8s.io/cli-runtime v0.29.0 // indirect
 	k8s.io/component-base v0.28.4 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum
index 42f1c5ca9..c5d2537de 100644
--- a/plugins/scheduler-k3s/go.sum
+++ b/plugins/scheduler-k3s/go.sum
@@ -33,16 +33,12 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
-github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
@@ -430,8 +426,6 @@ github.com/traefik/traefik/v2 v2.10.7 h1:vs91abMRVFTxBjn9hsLkqPbQ+D0oM1+bf4K8nD5
 github.com/traefik/traefik/v2 v2.10.7/go.mod h1:TD3saYNYbcBa7qDm29fIeyaiCMVX3ltv2YuaX3ANcnc=
 github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
-github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
-github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

From 204517074bfb0d3725dedbcc04684ad954d91674 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 03:26:32 -0500
Subject: [PATCH 094/113] fix: allow setting a custom ip address as the ip pf
 the dokku server

This will provide the ability to override cases where the detected ip address is invalid for whatever reason.
---
 docs/deployment/schedulers/k3s.md             | 24 ++++++-
 plugins/scheduler-k3s/functions.go            | 31 +++++++++
 .../src/subcommands/subcommands.go            |  6 +-
 plugins/scheduler-k3s/subcommands.go          | 67 +++++--------------
 4 files changed, 72 insertions(+), 56 deletions(-)

diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md
index c39076768..068a2e0e7 100644
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -41,6 +41,12 @@ By default, the k3s installation can run both app and system workloads. For clus
 dokku scheduler-k3s:initialize --taint-scheduling
 ```
 
+By default, Dokku will attempt to auto-detect the IP address of the server. In cases where the auto-detected IP address is incorrect, an override may be specified via the `--server-ip` flag:
+
+```shell
+dokku scheduler-k3s:initialize --server-ip 192.168.20.15
+```
+
 ### Adding nodes to the cluster
 
 > [!WARNING]
@@ -60,6 +66,12 @@ If the server isn't in the `known_hosts` file, the connection will fail. This ca
 dokku scheduler-k3s:cluster-add --insecure-allow-unknown-hosts ssh://root@worker-1.example.com
 ```
 
+By default, Dokku will attempt to auto-detect the IP address of the Dokku server for the remote server to connect to. In cases where the auto-detected IP address is incorrect, an override may be specified via the `--server-ip` flag:
+
+```shell
+dokku scheduler-k3s:cluster-add --server-ip 192.168.20.15 ssh://root@worker-1.example.com
+```
+
 #### Adding a server node
 
 > [!NOTE]
@@ -76,13 +88,19 @@ dokku scheduler-k3s:cluster-add  --role server ssh://root@server-1.example.com
 Server nodes allow any workloads to be scheduled on them by default, in addition to the control-plane, etcd, and the scheduler itself. To avoid app workloads being scheduled on your control-plane, use the `--taint-scheduling` flag:
 
 ```shell
-dokku scheduler-k3s:cluster-add --role server --taint-scheduling ssh://root@cluster-1.example.com
+dokku scheduler-k3s:cluster-add --role server --taint-scheduling ssh://root@server-1.example.com
 ```
 
 If the server isn't in the `known_hosts` file, the connection will fail. This can be bypassed by setting the `--insecure-allow-unknown-hosts` flag:
 
 ```shell
-dokku scheduler-k3s:cluster-add --role server --insecure-allow-unknown-hosts ssh://root@worker-1.example.com
+dokku scheduler-k3s:cluster-add --role server --insecure-allow-unknown-hosts ssh://root@server-1.example.com
+```
+
+By default, Dokku will attempt to auto-detect the IP address of the Dokku server for the remote server to connect to. In cases where the auto-detected IP address is incorrect, an override may be specified via the `--server-ip` flag:
+
+```shell
+dokku scheduler-k3s:cluster-add --role server --server-ip 192.168.20.15 ssh://root@server-1.example.com
 ```
 
 #### Changing the network interface
@@ -362,7 +380,7 @@ dokku scheduler-k3s:cluster-add  --role server ssh://root@server-1.example.com
 Server nodes allow any workloads to be scheduled on them by default, in addition to the control-plane, etcd, and the scheduler itself. To avoid app workloads being scheduled on your control-plane, use the `--taint-scheduling` flag:
 
 ```shell
-dokku scheduler-k3s:cluster-add --role server --taint-scheduling ssh://root@cluster-1.example.com
+dokku scheduler-k3s:cluster-add --role server --taint-scheduling ssh://root@server-1.example.com
 ```
 
 If the server isn't in the `known_hosts` file, the connection will fail. This can be bypassed by setting the `--insecure-allow-unknown-hosts` flag:
diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index e7387bbac..1406a430b 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"os"
 	"sort"
 	"strconv"
@@ -502,6 +503,36 @@ func getProcessResources(appName string, processType string) (ProcessResourcesMa
 	return processResources, nil
 }
 
+func getServerIP() (string, error) {
+	serverIP := ""
+	networkInterface := getGlobalNetworkInterface()
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return "", fmt.Errorf("Unable to get network interfaces: %w", err)
+	}
+
+	for _, iface := range ifaces {
+		if iface.Name == networkInterface {
+			addr, err := iface.Addrs()
+			if err != nil {
+				return "", fmt.Errorf("Unable to get network addresses for interface %s: %w", networkInterface, err)
+			}
+			for _, a := range addr {
+				if ipnet, ok := a.(*net.IPNet); ok {
+					if ipnet.IP.To4() != nil {
+						serverIP = ipnet.IP.String()
+					}
+				}
+			}
+		}
+	}
+
+	if len(serverIP) == 0 {
+		return "", fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface))
+	}
+	return serverIP, nil
+}
+
 func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
 	command := extractStartCommand(input)
 	fields, err := shell.Fields(command, func(name string) string {
diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go
index 60bf79e23..38616f7d5 100644
--- a/plugins/scheduler-k3s/src/subcommands/subcommands.go
+++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go
@@ -21,16 +21,18 @@ func main() {
 	case "initialize":
 		args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
+		serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandInitialize(*taintScheduling)
+		err = scheduler_k3s.CommandInitialize(*serverIP, *taintScheduling)
 	case "cluster-add":
 		args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError)
 		allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts")
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
+		serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
 		role := args.String("role", "worker", "role: [ server | worker ]")
 		args.Parse(os.Args[2:])
 		remoteHost := args.Arg(0)
-		err = scheduler_k3s.CommandClusterAdd(*role, remoteHost, *allowUknownHosts, *taintScheduling)
+		err = scheduler_k3s.CommandClusterAdd(*role, remoteHost, *serverIP, *allowUknownHosts, *taintScheduling)
 	case "cluster-list":
 		args := flag.NewFlagSet("scheduler-k3s:cluster-list", flag.ExitOnError)
 		format := args.String("format", "stdout", "format: [ stdout | json ]")
diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index ee1a2b0c2..30d347b4b 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
-	"net"
 	"net/url"
 	"os"
 	"os/signal"
@@ -18,7 +17,7 @@ import (
 )
 
 // CommandInitialize initializes a k3s cluster on the local server
-func CommandInitialize(taintScheduling bool) error {
+func CommandInitialize(serverIP string, taintScheduling bool) error {
 	if err := isK3sInstalled(); err == nil {
 		return fmt.Errorf("k3s already installed, cannot re-initialize k3s")
 	}
@@ -34,31 +33,14 @@ func CommandInitialize(taintScheduling bool) error {
 		cancel()
 	}()
 
-	networkInterface := getGlobalNetworkInterface()
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return fmt.Errorf("Unable to get network interfaces: %w", err)
-	}
-
-	serverIp := ""
-	for _, iface := range ifaces {
-		if iface.Name == networkInterface {
-			addr, err := iface.Addrs()
-			if err != nil {
-				return fmt.Errorf("Unable to get network addresses for interface %s: %w", networkInterface, err)
-			}
-			for _, a := range addr {
-				if ipnet, ok := a.(*net.IPNet); ok {
-					if ipnet.IP.To4() != nil {
-						serverIp = ipnet.IP.String()
-					}
-				}
-			}
+	if serverIP == "" {
+		var err error
+		serverIP, err = getServerIP()
+		if err != nil {
+			return fmt.Errorf("Unable to get server ip address: %w", err)
 		}
-	}
 
-	if len(serverIp) == 0 {
-		return fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface))
+		common.LogVerboseQuiet(fmt.Sprintf("Using server ip address: %s", serverIP))
 	}
 
 	common.LogInfo1Quiet("Initializing k3s")
@@ -156,7 +138,7 @@ func CommandInitialize(taintScheduling bool) error {
 		}
 	}
 
-	nodeName := serverIp
+	nodeName := serverIP
 	n := 5
 	b := make([]byte, n)
 	if _, err := rand.Read(b); err != nil {
@@ -280,7 +262,7 @@ func CommandInitialize(taintScheduling bool) error {
 }
 
 // CommandClusterAdd adds a server to the k3s cluster
-func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, taintScheduling bool) error {
+func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUknownHosts bool, taintScheduling bool) error {
 	if err := isK3sInstalled(); err != nil {
 		return fmt.Errorf("k3s not installed, cannot join cluster")
 	}
@@ -298,31 +280,14 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		return fmt.Errorf("Taint scheduling can only be used on the server role")
 	}
 
-	networkInterface := getGlobalNetworkInterface()
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return fmt.Errorf("Unable to get network interfaces: %w", err)
-	}
-
-	serverIp := ""
-	for _, iface := range ifaces {
-		if iface.Name == networkInterface {
-			addr, err := iface.Addrs()
-			if err != nil {
-				return fmt.Errorf("Unable to get network addresses for interface %s: %w", networkInterface, err)
-			}
-			for _, a := range addr {
-				if ipnet, ok := a.(*net.IPNet); ok {
-					if ipnet.IP.To4() != nil {
-						serverIp = ipnet.IP.String()
-					}
-				}
-			}
+	if serverIP == "" {
+		var err error
+		serverIP, err = getServerIP()
+		if err != nil {
+			return fmt.Errorf("Unable to get server ip address: %w", err)
 		}
-	}
 
-	if len(serverIp) == 0 {
-		return fmt.Errorf(fmt.Sprintf("Unable to determine server ip address from network-interface %s", networkInterface))
+		common.LogVerboseQuiet(fmt.Sprintf("Using server ip address: %s", serverIP))
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -464,7 +429,7 @@ func CommandClusterAdd(role string, remoteHost string, allowUknownHosts bool, ta
 		"--node-name", nodeName,
 		// server to connect to as the main
 		"--server",
-		fmt.Sprintf("https://%s:6443", serverIp),
+		fmt.Sprintf("https://%s:6443", serverIP),
 		// specify a token
 		"--token",
 		token,

From ff849cd4f848b7530e42567aead8c80ae92e8a17 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 04:25:25 -0500
Subject: [PATCH 095/113] fix: touch the known hosts file earlier

---
 plugins/common/ssh.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/plugins/common/ssh.go b/plugins/common/ssh.go
index 0c405d448..7d26e089b 100644
--- a/plugins/common/ssh.go
+++ b/plugins/common/ssh.go
@@ -244,6 +244,10 @@ func (task SshTask) Execute(ctx context.Context) (SshResult, error) {
 		task.Port = 22
 	}
 
+	if err := TouchFile(filepath.Join(os.Getenv("DOKKU_ROOT"), ".ssh", "known_hosts")); err != nil {
+		return SshResult{}, fmt.Errorf("failed to touch known_hosts file: %w", err)
+	}
+
 	auth, err := goph.Key(task.SshKeyPath, "")
 	if err != nil {
 		return SshResult{}, fmt.Errorf("failed to load ssh key: %w", err)
@@ -254,10 +258,6 @@ func (task SshTask) Execute(ctx context.Context) (SshResult, error) {
 		return SshResult{}, fmt.Errorf("failed to load known hosts: %w", err)
 	}
 
-	if err := TouchFile(filepath.Join(os.Getenv("DOKKU_ROOT"), ".ssh", "known_hosts")); err != nil {
-		return SshResult{}, fmt.Errorf("failed to touch known_hosts file: %w", err)
-	}
-
 	if task.AllowUknownHosts {
 		callback = ssh.InsecureIgnoreHostKey()
 	}

From 57e76e29d18845d6189cd6607a833ea4fed7f1ad Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 04:25:41 -0500
Subject: [PATCH 096/113] fix: correct the scope on the token

---
 plugins/scheduler-k3s/subcommands.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go
index 30d347b4b..7bece9a73 100644
--- a/plugins/scheduler-k3s/subcommands.go
+++ b/plugins/scheduler-k3s/subcommands.go
@@ -132,7 +132,7 @@ func CommandInitialize(serverIP string, taintScheduling bool) error {
 			return fmt.Errorf("Unable to generate random node name: %w", err)
 		}
 
-		token := strings.ToLower(fmt.Sprintf("%X", b))
+		token = strings.ToLower(fmt.Sprintf("%X", b))
 		if err := CommandSet("--global", "token", token); err != nil {
 			return fmt.Errorf("Unable to set k3s token: %w", err)
 		}

From a0811c2cb3e71668e9c5f122ad7587ffe6d6eb87 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 04:42:20 -0500
Subject: [PATCH 097/113] fix: move field to correct location

---
 plugins/scheduler-k3s/templates/chart/cron-job.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
index 13fbf4109..6e813cc12 100644
--- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml
+++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
@@ -21,6 +21,7 @@ metadata:
 spec:
   concurrencyPolicy: Allow
   failedJobsHistoryLimit: 10
+  ttlSecondsAfterFinished: 60
   jobTemplate:
     metadata:
       annotations:
@@ -95,9 +96,8 @@ spec:
           imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }}
           {{- end }}
           restartPolicy: Never
-          ttlSecondsAfterFinished: 60
   schedule: {{ $config.cron.schedule }}
   startingDeadlineSeconds: 60
   successfulJobsHistoryLimit: 10
   suspend: false
-  timeZone: Etc/UTC
\ No newline at end of file
+  timeZone: Etc/UTC

From fbbc967e8dab684c143d4c2acf696a9ad94a0e00 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 04:55:24 -0500
Subject: [PATCH 098/113] chore: add todo

---
 plugins/scheduler-k3s/report.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go
index b5a7672fe..601918c52 100644
--- a/plugins/scheduler-k3s/report.go
+++ b/plugins/scheduler-k3s/report.go
@@ -10,6 +10,7 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		return err
 	}
 
+	// todo: add all fields
 	flags := map[string]common.ReportFunc{}
 
 	flagKeys := []string{}

From cc62bc080be37e5705849f26fb24b81e04783505 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 04:55:38 -0500
Subject: [PATCH 099/113] fix: use correct field for comparison

---
 plugins/scheduler-k3s/templates/chart/ingress-route.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
index 444a25c88..5b48bb806 100644
--- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
+++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
@@ -17,7 +17,7 @@ metadata:
 spec:
   entryPoints:
   - web
-  {{- if and $config.web.tls (eq $port_map "https") }}
+  {{- if and $config.web.tls (eq $port_map.scheme "https") }}
   - websecure
   {{- end }}
   routes:

From a9e6e966977c1fc2a5226062ea1d2fd0033ee8c0 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 05:01:23 -0500
Subject: [PATCH 100/113] feat: implement scheduler-k3s:report

---
 plugins/scheduler-k3s/functions.go |  6 +-
 plugins/scheduler-k3s/report.go    | 94 +++++++++++++++++++++++++++++-
 2 files changed, 95 insertions(+), 5 deletions(-)

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 1406a430b..075fc2a34 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -295,14 +295,14 @@ func getLetsencryptServer(appName string) string {
 	return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "")
 }
 
-func getGlobalLetsencryptServer(appName string) string {
-	return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "prod")
+func getGlobalLetsencryptServer() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "letsencrypt-server", "prod")
 }
 
 func getComputedLetsencryptServer(appName string) string {
 	letsencryptServer := getLetsencryptServer(appName)
 	if letsencryptServer == "" {
-		letsencryptServer = getGlobalLetsencryptServer(appName)
+		letsencryptServer = getGlobalLetsencryptServer()
 	}
 
 	return letsencryptServer
diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go
index 601918c52..1974114fa 100644
--- a/plugins/scheduler-k3s/report.go
+++ b/plugins/scheduler-k3s/report.go
@@ -10,8 +10,26 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		return err
 	}
 
-	// todo: add all fields
-	flags := map[string]common.ReportFunc{}
+	flags := map[string]common.ReportFunc{
+		"--scheduler-k3s-computed-deploy-timeout":       reportComputedDeployTimeout,
+		"--scheduler-k3s-deploy-timeout":                reportDeployTimeout,
+		"--scheduler-k3s-global-deploy-timeout":         reportGlobalDeployTimeout,
+		"--scheduler-k3s-computed-image-pull-secrets":   reportComputedImagePullSecrets,
+		"--scheduler-k3s-image-pull-secrets":            reportImagePullSecrets,
+		"--scheduler-k3s-global-image-pull-secrets":     reportGlobalImagePullSecrets,
+		"--scheduler-k3s-computed-letsencrypt-server":   reportComputedLetsencryptServer,
+		"--scheduler-k3s-letsencrypt-server":            reportLetsencryptServer,
+		"--scheduler-k3s-global-letsencrypt-server":     reportGlobalLetsencryptServer,
+		"--scheduler-k3s-global-letsencrypt-email-prod": reportGlobalLetsencryptEmailProd,
+		"--scheduler-k3s-global-letsencrypt-email-stag": reportGlobalLetsencryptEmailStag,
+		"--scheduler-k3s-computed-namespace":            reportComputedNamespace,
+		"--scheduler-k3s-namespace":                     reportNamespace,
+		"--scheduler-k3s-global-namespace":              reportGlobalNamespace,
+		"--scheduler-k3s-global-network-interface":      reportGlobalNetworkInterface,
+		"--scheduler-k3s-computed-rollback-on-failure":  reportComputedRollbackOnFailure,
+		"--scheduler-k3s-rollback-on-failure":           reportRollbackOnFailure,
+		"--scheduler-k3s-global-rollback-on-failure":    reportGlobalRollbackOnFailure,
+	}
 
 	flagKeys := []string{}
 	for flagKey := range flags {
@@ -23,3 +41,75 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 	infoFlags := common.CollectReport(appName, infoFlag, flags)
 	return common.ReportSingleApp("scheduler-k3s", appName, infoFlag, infoFlags, flagKeys, format, trimPrefix, uppercaseFirstCharacter)
 }
+
+func reportComputedDeployTimeout(appName string) string {
+	return getComputedDeployTimeout(appName)
+}
+
+func reportDeployTimeout(appName string) string {
+	return getDeployTimeout(appName)
+}
+
+func reportGlobalDeployTimeout(appName string) string {
+	return getGlobalDeployTimeout()
+}
+
+func reportComputedImagePullSecrets(appName string) string {
+	return getComputedImagePullSecrets(appName)
+}
+
+func reportImagePullSecrets(appName string) string {
+	return getImagePullSecrets(appName)
+}
+
+func reportGlobalImagePullSecrets(appName string) string {
+	return getGlobalImagePullSecrets()
+}
+
+func reportComputedLetsencryptServer(appName string) string {
+	return getComputedLetsencryptServer(appName)
+}
+
+func reportLetsencryptServer(appName string) string {
+	return getLetsencryptServer(appName)
+}
+
+func reportGlobalLetsencryptServer(appName string) string {
+	return getGlobalLetsencryptServer()
+}
+
+func reportGlobalLetsencryptEmailProd(appName string) string {
+	return getGlobalLetsencryptEmailProd()
+}
+
+func reportGlobalLetsencryptEmailStag(appName string) string {
+	return getGlobalLetsencryptEmailStag()
+}
+
+func reportComputedNamespace(appName string) string {
+	return getComputedNamespace(appName)
+}
+
+func reportNamespace(appName string) string {
+	return getNamespace(appName)
+}
+
+func reportGlobalNamespace(appName string) string {
+	return getGlobalNamespace()
+}
+
+func reportGlobalNetworkInterface(appName string) string {
+	return getGlobalNetworkInterface()
+}
+
+func reportComputedRollbackOnFailure(appName string) string {
+	return getComputedRollbackOnFailure(appName)
+}
+
+func reportRollbackOnFailure(appName string) string {
+	return getRollbackOnFailure(appName)
+}
+
+func reportGlobalRollbackOnFailure(appName string) string {
+	return getGlobalRollbackOnFailure()
+}

From c8f5ad9029708fa6462696cdeef6fb99b9e1163a Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 06:18:25 -0500
Subject: [PATCH 101/113] fix: correct issues in certificate generation

---
 plugins/scheduler-k3s/template.go                      | 2 +-
 plugins/scheduler-k3s/templates/chart/certificate.yaml | 7 +++----
 plugins/scheduler-k3s/triggers.go                      | 5 ++++-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 03cc63ade..2dced9de1 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -108,7 +108,7 @@ const (
 type ProcessWeb struct {
 	Domains  []string         `yaml:"domains,omitempty"`
 	PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"`
-	TLS      bool             `yaml:"tls"`
+	TLS      ProcessTls       `yaml:"tls"`
 }
 
 type ProcessResourcesMap struct {
diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml
index 30c385042..4bb8603c7 100644
--- a/plugins/scheduler-k3s/templates/chart/certificate.yaml
+++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml
@@ -1,6 +1,6 @@
 {{- $processName := "PROCESS_NAME" }}
 {{- $config := .Values.processes.PROCESS_NAME }}
-{{- if and $config.web.tls $config.domains }}
+{{- if and $config.web.tls.enabled $config.web.domains }}
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
@@ -16,7 +16,7 @@ metadata:
 spec:
   issuerRef:
     kind: ClusterIssuer
-    name: {{ $config.tls.issuer_name }}
+    name: {{ $config.web.tls.issuer_name }}
   secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
   secretTemplate:
     annotations:
@@ -24,9 +24,8 @@ spec:
     labels:
       app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }}
       app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
-status: {}
   dnsNames:
-    {{- range $config.domains }}
+    {{- range $config.web.domains }}
     - {{ . }}
     {{- end }}
 {{- end }}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index e5ec1492c..78e7b310d 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -378,7 +378,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			processValues.Web = ProcessWeb{
 				Domains:  domains,
 				PortMaps: []ProcessPortMap{},
-				TLS:      tls,
+				TLS: ProcessTls{
+					Enabled:    tls,
+					IssuerName: issuerName,
+				},
 			}
 			processValues.ProcessType = ProcessType_Web
 			for _, portMap := range portMaps {

From 4a7db5ce2491307f335e72cd4548424482346607 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 07:25:03 -0500
Subject: [PATCH 102/113] feat: apply the cluster issuers on every app deploy

This ensures we can always get certificates regardless of what app gets deployed with whatever configuration.
---
 plugins/scheduler-k3s/functions.go            | 118 ++++++++++++++++++
 plugins/scheduler-k3s/template.go             |  73 ++---------
 .../templates/chart/cluster-issuer.yaml       |  19 +++
 plugins/scheduler-k3s/triggers.go             |  25 ++--
 4 files changed, 154 insertions(+), 81 deletions(-)
 create mode 100644 plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml

diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go
index 075fc2a34..d55398cb1 100644
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -114,6 +115,123 @@ type WaitForPodToExistInput struct {
 	LabelSelector string
 }
 
+func applyClusterIssuers(ctx context.Context, appName string) (string, bool, error) {
+	chartDir, err := os.MkdirTemp("", "cluster-issuer-chart-")
+	if err != nil {
+		return "", false, fmt.Errorf("Error creating cluster-issuer chart directory: %w", err)
+	}
+	defer os.RemoveAll(chartDir)
+
+	// create the chart.yaml
+	chart := &Chart{
+		ApiVersion: "v2",
+		AppVersion: "1.0.0",
+		Name:       appName,
+		Version:    "0.0.1",
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: chart,
+		Path:   filepath.Join(chartDir, "Chart.yaml"),
+	})
+	if err != nil {
+		return "", false, fmt.Errorf("Error writing cluster-issuer chart: %w", err)
+	}
+
+	// create the values.yaml
+	issuerName := "letsencrypt-stag"
+	server := getComputedLetsencryptServer(appName)
+	if server == "prod" || server == "production" {
+		issuerName = "letsencrypt-prod"
+	} else if server != "stag" && server != "staging" {
+		return "", false, fmt.Errorf("Invalid letsencrypt server config: %s", server)
+	}
+
+	tls := false
+	letsencryptEmailStag := getGlobalLetsencryptEmailStag()
+	letsencryptEmailProd := getGlobalLetsencryptEmailProd()
+	if issuerName == "letsencrypt-stag" {
+		tls = letsencryptEmailStag != ""
+	}
+	if issuerName == "letsencrypt-prod" {
+		tls = letsencryptEmailProd != ""
+	}
+
+	clusterIssuerValues := ClusterIssuerValues{
+		ClusterIssuers: map[string]ClusterIssuer{
+			"letsencrypt-stag": {
+				Email:   letsencryptEmailStag,
+				Enabled: letsencryptEmailStag != "",
+				Name:    "letsencrypt-stag",
+				Server:  "https://acme-staging-v02.api.letsencrypt.org/directory",
+			},
+			"letsencrypt-prod": {
+				Email:   letsencryptEmailProd,
+				Enabled: letsencryptEmailProd != "",
+				Name:    "letsencrypt-prod",
+				Server:  "https://acme-v02.api.letsencrypt.org/directory",
+			},
+		},
+	}
+
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
+		return "", false, fmt.Errorf("Error creating cluster-issuer chart templates directory: %w", err)
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: clusterIssuerValues,
+		Path:   filepath.Join(chartDir, "values.yaml"),
+	})
+	if err != nil {
+		return "", false, fmt.Errorf("Error writing chart: %w", err)
+	}
+
+	// create the templates/cluster-issuer.yaml
+	b, err := templates.ReadFile("templates/chart/cluster-issuer.yaml")
+	if err != nil {
+		return "", false, fmt.Errorf("Error reading cluster-issuer template: %w", err)
+	}
+
+	filename := filepath.Join(chartDir, "templates", "cluster-issuer.yaml")
+	err = os.WriteFile(filename, b, os.FileMode(0644))
+	if err != nil {
+		return "", false, fmt.Errorf("Error writing cluster-issuer template: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(filename)
+	}
+
+	// install the chart
+	helmAgent, err := NewHelmAgent("cert-manager", DevNullPrinter)
+	if err != nil {
+		return issuerName, tls, fmt.Errorf("Error creating helm agent: %w", err)
+	}
+
+	chartPath, err := filepath.Abs(chartDir)
+	if err != nil {
+		return issuerName, tls, fmt.Errorf("Error getting chart path: %w", err)
+	}
+
+	timeoutDuration, err := time.ParseDuration("300s")
+	if err != nil {
+		return issuerName, tls, fmt.Errorf("Error parsing deploy timeout duration: %w", err)
+	}
+
+	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
+		ChartPath:         chartPath,
+		Namespace:         "cert-manager",
+		ReleaseName:       "cluster-issuers",
+		RollbackOnFailure: true,
+		Timeout:           timeoutDuration,
+	})
+	if err != nil {
+		return issuerName, tls, fmt.Errorf("Error installing cluster-issuer chart: %w", err)
+	}
+
+	return issuerName, tls, nil
+}
+
 func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
 	clientset, err := NewKubernetesClient()
 	if err != nil {
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 2dced9de1..766623326 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -6,9 +6,6 @@ import (
 	"os"
 	"strings"
 
-	acmev1 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
-	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
-	certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
 	"github.com/dokku/dokku/plugins/common"
 	"gopkg.in/yaml.v3"
 	batchv1 "k8s.io/api/batch/v1"
@@ -25,7 +22,11 @@ type Chart struct {
 	Version    string `yaml:"version"`
 }
 
-type Values struct {
+type ClusterIssuerValues struct {
+	ClusterIssuers map[string]ClusterIssuer `yaml:"cluster_issuers"`
+}
+
+type AppValues struct {
 	Global    GlobalValues             `yaml:"global"`
 	Processes map[string]ProcessValues `yaml:"processes"`
 }
@@ -164,65 +165,11 @@ type ProcessTls struct {
 }
 
 type ClusterIssuer struct {
-	Email        string
-	IngressClass string
-	Name         string
-	Namespace    string
-	Server       string
-}
-
-func templateKubernetesClusterIssuer(input ClusterIssuer) (certmanagerv1.ClusterIssuer, error) {
-	if input.Email == "" {
-		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Email cannot be empty")
-	}
-	if input.Server == "" {
-		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Server cannot be empty")
-	}
-	if input.Name == "" {
-		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Name cannot be empty")
-	}
-	if input.Namespace == "" {
-		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Namespace cannot be empty")
-	}
-
-	if input.Server == "staging" {
-		input.Server = "https://acme-staging-v02.api.letsencrypt.org/directory"
-	} else if input.Server == "production" {
-		input.Server = "https://acme-v02.api.letsencrypt.org/directory"
-	} else {
-		return certmanagerv1.ClusterIssuer{}, fmt.Errorf("Server must be either staging or production")
-	}
-
-	clusterIssuer := certmanagerv1.ClusterIssuer{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      input.Name,
-			Namespace: input.Namespace,
-		},
-		Spec: certmanagerv1.IssuerSpec{
-			IssuerConfig: certmanagerv1.IssuerConfig{
-				ACME: &acmev1.ACMEIssuer{
-					Email:  input.Email,
-					Server: input.Server,
-					PrivateKey: certmanagermetav1.SecretKeySelector{
-						LocalObjectReference: certmanagermetav1.LocalObjectReference{
-							Name: input.Name,
-						},
-					},
-					Solvers: []acmev1.ACMEChallengeSolver{
-						{
-							HTTP01: &acmev1.ACMEChallengeSolverHTTP01{
-								Ingress: &acmev1.ACMEChallengeSolverHTTP01Ingress{
-									Class: ptr.To(input.IngressClass),
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	return clusterIssuer, nil
+	Email        string `yaml:"email"`
+	Enabled      bool   `yaml:"enabled"`
+	IngressClass string `yaml:"ingress_class"`
+	Name         string `yaml:"name"`
+	Server       string `yaml:"server"`
 }
 
 type Job struct {
diff --git a/plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml b/plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml
new file mode 100644
index 000000000..30fe0a2f7
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml
@@ -0,0 +1,19 @@
+{{- range $name, $config := .Values.cluster_issuers }}
+{{- if $config.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: {{ $config.name }}
+spec:
+  acme:
+    email: {{ $config.email }}
+    server: {{ $config.server }}
+    privateKeySecretRef:
+      name: {{ $config.name }}
+    solvers:
+    - http01:
+        ingress:
+          class: {{ $config.ingress_class }}
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 78e7b310d..4be8e71d4 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -222,6 +222,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error loading environment for deployment: %w", err)
 	}
 
+	issuerName, tlsEnabled, err := applyClusterIssuers(ctx, appName)
+	if err != nil {
+		return fmt.Errorf("Error applying cluster issuers: %w", err)
+	}
+
 	chartDir, err := os.MkdirTemp("", "dokku-chart-")
 	if err != nil {
 		return fmt.Errorf("Error creating chart directory: %w", err)
@@ -273,22 +278,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error fetching cron entries: %w", err)
 	}
 
-	issuerName := "letsencrypt-stag"
-	server := getComputedLetsencryptServer(appName)
-	if server == "prod" || server == "production" {
-		issuerName = "letsencrypt-prod"
-	} else if server != "stag" && server != "staging" {
-		return fmt.Errorf("Invalid letsencrypt server config: %s", server)
-	}
-
-	tls := false
-	if issuerName == "letsencrypt-stag" {
-		tls = getGlobalLetsencryptEmailStag() != ""
-	}
-	if issuerName == "letsencrypt-prod" {
-		tls = getGlobalLetsencryptEmailProd() != ""
-	}
-
 	domains := []string{}
 	if _, ok := processes["web"]; ok {
 		err = common.PlugnTrigger("domains-vhost-enabled", []string{appName}...)
@@ -322,7 +311,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error writing chart: %w", err)
 	}
 
-	values := &Values{
+	values := &AppValues{
 		Global: GlobalValues{
 			AppName:      appName,
 			DeploymentID: fmt.Sprint(deploymentId),
@@ -379,7 +368,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 				Domains:  domains,
 				PortMaps: []ProcessPortMap{},
 				TLS: ProcessTls{
-					Enabled:    tls,
+					Enabled:    tlsEnabled,
 					IssuerName: issuerName,
 				},
 			}

From 25925c0ae2f490bbe8db6a78eb1fd3212631ce01 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 07:25:18 -0500
Subject: [PATCH 103/113] chore: remove logging from inside helm client

---
 plugins/scheduler-k3s/helm.go     | 2 --
 plugins/scheduler-k3s/triggers.go | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go
index 94b176326..bf16fa1dc 100644
--- a/plugins/scheduler-k3s/helm.go
+++ b/plugins/scheduler-k3s/helm.go
@@ -207,11 +207,9 @@ func (h *HelmAgent) InstallOrUpgradeChart(ctx context.Context, input ChartInput)
 	}
 
 	if chartExists {
-		common.LogExclaim(fmt.Sprintf("Upgrading %s", input.ReleaseName))
 		return h.UpgradeChart(ctx, input)
 	}
 
-	common.LogExclaim(fmt.Sprintf("Installing %s", input.ReleaseName))
 	return h.InstallChart(ctx, input)
 }
 
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 4be8e71d4..183b68ef7 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -515,6 +515,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
 	}
 
+	common.LogExclaim(fmt.Sprintf("Installing %s", appName))
 	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
 		ChartPath:         chartPath,
 		Namespace:         namespace,

From 99f20a962ec8d39dcd486742441b4587cd637db9 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 07:25:28 -0500
Subject: [PATCH 104/113] refactor: rename helm chart to app name

---
 plugins/scheduler-k3s/triggers.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 183b68ef7..612e08ffd 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -519,7 +519,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
 		ChartPath:         chartPath,
 		Namespace:         namespace,
-		ReleaseName:       fmt.Sprintf("dokku-%s", appName),
+		ReleaseName:       appName,
 		RollbackOnFailure: allowRollbacks,
 		Timeout:           timeoutDuration,
 	})

From cf7d35601d59000ba81afee7eb744776f477f893 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 07:25:42 -0500
Subject: [PATCH 105/113] fix: delete the temporary helm chart on disk

---
 plugins/scheduler-k3s/triggers.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index 612e08ffd..da93f8037 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -231,7 +231,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	if err != nil {
 		return fmt.Errorf("Error creating chart directory: %w", err)
 	}
-	// defer os.RemoveAll(chartDir)
+	defer os.RemoveAll(chartDir)
 
 	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
 		return fmt.Errorf("Error creating chart templates directory: %w", err)

From 1d07801f3cba7f03553b582a8db6775e7f0568d2 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 07:25:51 -0500
Subject: [PATCH 106/113] fix: set ttl property at correct level for cronjobs

---
 plugins/scheduler-k3s/templates/chart/cron-job.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
index 6e813cc12..ff5076c8d 100644
--- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml
+++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
@@ -21,7 +21,6 @@ metadata:
 spec:
   concurrencyPolicy: Allow
   failedJobsHistoryLimit: 10
-  ttlSecondsAfterFinished: 60
   jobTemplate:
     metadata:
       annotations:
@@ -40,6 +39,7 @@ spec:
     spec:
       backoffLimit: 0
       podReplacementPolicy: Failed
+      ttlSecondsAfterFinished: 60
       template:
         metadata:
           annotations:

From 7b2484962b730c3ec8b6967d9ef157db8d4b0a44 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 07:59:14 -0500
Subject: [PATCH 107/113] fix: set tls field on ingress-route

---
 plugins/scheduler-k3s/templates/chart/ingress-route.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
index 5b48bb806..cb306f81d 100644
--- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
+++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
@@ -31,5 +31,9 @@ spec:
         port: {{ $port_map.name }}
         scheme: {{ $port_map.scheme }}
     {{- end }}
+  {{- if and $config.web.tls (eq $port_map.scheme "https") }}
+  tls:
+    secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
+  {{- end }}
 {{- end }}
 {{- end }}

From d8fcabdf403cb01c0fc934af99369569af8b5add Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 08:10:16 -0500
Subject: [PATCH 108/113] fix: always route to http instead of https

Otherwise, we're assuming the app pod is going to handle https requests and not the ingress
---
 plugins/scheduler-k3s/templates/chart/ingress-route.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
index cb306f81d..1efefa8e8 100644
--- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
+++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
@@ -29,7 +29,7 @@ spec:
         namespace: {{ $.Values.global.namespace }}
         passHostHeader: true
         port: {{ $port_map.name }}
-        scheme: {{ $port_map.scheme }}
+        scheme: http
     {{- end }}
   {{- if and $config.web.tls (eq $port_map.scheme "https") }}
   tls:

From 1f35575a2a12be55514a6a0ea39e21a3654eee6f Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 08:59:44 -0500
Subject: [PATCH 109/113] fix: use the container port as the target to fix an
 issue where kubernetes will remove duplicate containerPort entries with
 different names

---
 plugins/scheduler-k3s/templates/chart/service.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml
index a3ed18c83..4d9f33bf3 100644
--- a/plugins/scheduler-k3s/templates/chart/service.yaml
+++ b/plugins/scheduler-k3s/templates/chart/service.yaml
@@ -18,7 +18,7 @@ spec:
   - name: {{ $port_map.name }}
     port: {{ $port_map.host_port }}
     protocol: TCP
-    targetPort: {{ $port_map.name }}
+    targetPort: {{ $port_map.container_port }}
   {{- end }}
   selector:
     app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}

From c1b3c73096e06862ae02b07b4a6d9f478e89489d Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 09:32:41 -0500
Subject: [PATCH 110/113] chore: drop unnecessary key

---
 plugins/scheduler-k3s/templates/chart/certificate.yaml | 1 -
 plugins/scheduler-k3s/templates/chart/cron-job.yaml    | 2 --
 plugins/scheduler-k3s/templates/chart/deployment.yaml  | 2 --
 3 files changed, 5 deletions(-)

diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml
index 4bb8603c7..4feda021e 100644
--- a/plugins/scheduler-k3s/templates/chart/certificate.yaml
+++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml
@@ -7,7 +7,6 @@ kind: Certificate
 metadata:
   annotations:
     dokku.com/managed: "true"
-  creationTimestamp: null
   labels:
     app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }}
     app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
index ff5076c8d..d34e14ef3 100644
--- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml
+++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml
@@ -30,7 +30,6 @@ spec:
         dokku.com/job-suffix: {{ $config.cron.suffix }}
         dokku.com/managed: "true"
         kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
-      creationTimestamp: null
       labels:
         app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
         app.kubernetes.io/name: cron
@@ -49,7 +48,6 @@ spec:
             dokku.com/job-suffix: {{ $config.cron.suffix }}
             dokku.com/managed: "true"
             kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
-          creationTimestamp: null
           labels:
             app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
             app.kubernetes.io/name: cron
diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml
index 89c6fbb7e..f6d36f665 100644
--- a/plugins/scheduler-k3s/templates/chart/deployment.yaml
+++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml
@@ -9,7 +9,6 @@ metadata:
     dokku.com/builder-type: {{ $.Values.global.image.type }}
     dokku.com/managed: "true"
     kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
-  creationTimestamp: null
   labels:
     app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
     app.kubernetes.io/name: {{ $processName }}
@@ -32,7 +31,6 @@ spec:
         dokku.com/builder-type: {{ $.Values.global.image.type }}
         dokku.com/managed: "true"
         kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
-      creationTimestamp: null
       labels:
         app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
         app.kubernetes.io/name: {{ $processName }}

From 56ee8892952591fa171bbbca55eb02df08e24596 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 09:32:52 -0500
Subject: [PATCH 111/113] chore: note that the cluster-issuer is managed by
 dokku

---
 plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml b/plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml
index 30fe0a2f7..473136251 100644
--- a/plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml
+++ b/plugins/scheduler-k3s/templates/chart/cluster-issuer.yaml
@@ -4,6 +4,8 @@
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
+  annotations:
+    dokku.com/managed: "true"
   name: {{ $config.name }}
 spec:
   acme:

From 6707e5f1eb97314e51a35d003bc7368a0881fd29 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 09:45:20 -0500
Subject: [PATCH 112/113] feat: add redirect to https when both an http and
 https port mapping match

---
 plugins/scheduler-k3s/portmap.go              | 10 ++++----
 plugins/scheduler-k3s/template.go             | 11 +++++----
 .../chart/https-redirect-middleware.yaml      | 15 ++++++++++++
 .../templates/chart/ingress-route.yaml        | 12 +++++++---
 plugins/scheduler-k3s/triggers.go             | 23 ++++++++++++++-----
 5 files changed, 52 insertions(+), 19 deletions(-)
 create mode 100644 plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml

diff --git a/plugins/scheduler-k3s/portmap.go b/plugins/scheduler-k3s/portmap.go
index e4aa03807..6e012d888 100644
--- a/plugins/scheduler-k3s/portmap.go
+++ b/plugins/scheduler-k3s/portmap.go
@@ -25,27 +25,27 @@ func (p PortMap) IsAllowedHttps() bool {
 	return p.Scheme == "https" || p.ContainerPort == 443
 }
 
-func getPortMaps(appName string) ([]PortMap, error) {
+func getPortMaps(appName string) (map[string]PortMap, error) {
 	portMaps := []PortMap{}
 
+	allowedMappings := map[string]PortMap{}
 	output, err := common.PlugnTriggerOutputAsString("ports-get", []string{appName, "json"}...)
 	if err != nil {
-		return portMaps, err
+		return allowedMappings, err
 	}
 
 	err = json.Unmarshal([]byte(output), &portMaps)
 	if err != nil {
-		return portMaps, err
+		return allowedMappings, err
 	}
 
-	allowedMappings := []PortMap{}
 	for _, portMap := range portMaps {
 		if !portMap.IsAllowedHttp() && !portMap.IsAllowedHttps() {
 			// todo: log warning
 			continue
 		}
 
-		allowedMappings = append(allowedMappings, portMap)
+		allowedMappings[portMap.String()] = portMap
 	}
 
 	return allowedMappings, nil
diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go
index 766623326..5b0d94e81 100644
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -139,11 +139,12 @@ type ProcessCron struct {
 }
 
 type ProcessPortMap struct {
-	ContainerPort int32           `yaml:"container_port"`
-	HostPort      int32           `yaml:"host_port"`
-	Scheme        string          `yaml:"scheme"`
-	Protocol      PortmapProtocol `yaml:"protocol"`
-	Name          string          `yaml:"name"`
+	ContainerPort   int32           `yaml:"container_port"`
+	HostPort        int32           `yaml:"host_port"`
+	Scheme          string          `yaml:"scheme"`
+	Protocol        PortmapProtocol `yaml:"protocol"`
+	Name            string          `yaml:"name"`
+	RedirectToHttps bool            `yaml:"redirect_to_https"`
 }
 
 type PortmapProtocol string
diff --git a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml
new file mode 100644
index 000000000..4365c713c
--- /dev/null
+++ b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-redirect-to-https
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+  name: {{ $.Values.global.app_name}}-redirect-to-https
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  redirectScheme:
+    scheme: https
+    permanent: true
diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
index 1efefa8e8..1061eb114 100644
--- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
+++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
@@ -16,14 +16,20 @@ metadata:
   namespace: {{ $.Values.global.namespace }}
 spec:
   entryPoints:
-  - web
-  {{- if and $config.web.tls (eq $port_map.scheme "https") }}
+  {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }}
   - websecure
+  {{- else }}
+  - web
   {{- end }}
   routes:
     {{- range $ddx, $domain := $config.web.domains }}
     - kind: Rule
       match: Host(`{{ $domain }}`)
+      {{- if and $config.web.tls.enabled (and (eq $port_map.scheme "http") $port_map.redirect_to_https) }}
+      middlewares:
+        - name: {{ $.Values.global.app_name}}-redirect-to-https
+          namespace: traefik
+      {{- end }}
       services:
       - name: {{ $.Values.global.app_name }}-{{ $processName }}
         namespace: {{ $.Values.global.namespace }}
@@ -31,7 +37,7 @@ spec:
         port: {{ $port_map.name }}
         scheme: http
     {{- end }}
-  {{- if and $config.web.tls (eq $port_map.scheme "https") }}
+  {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }}
   tls:
     secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
   {{- end }}
diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go
index da93f8037..aed80782e 100644
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -372,18 +372,25 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 					IssuerName: issuerName,
 				},
 			}
+
 			processValues.ProcessType = ProcessType_Web
 			for _, portMap := range portMaps {
 				protocol := PortmapProtocol_TCP
 				if portMap.Scheme == "udp" {
 					protocol = PortmapProtocol_UDP
 				}
+
+				redirecToHttps := false
+				if _, ok := portMaps[fmt.Sprintf("https-443-%d", portMap.ContainerPort)]; ok && portMap.Scheme == "http" {
+					redirecToHttps = true
+				}
 				processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
-					ContainerPort: portMap.ContainerPort,
-					HostPort:      portMap.HostPort,
-					Name:          portMap.String(),
-					Protocol:      protocol,
-					Scheme:        portMap.Scheme,
+					ContainerPort:   portMap.ContainerPort,
+					HostPort:        portMap.HostPort,
+					Name:            portMap.String(),
+					Protocol:        protocol,
+					RedirectToHttps: redirecToHttps,
+					Scheme:          portMap.Scheme,
 				})
 			}
 
@@ -393,7 +400,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 
 		values.Processes[processType] = processValues
 
-		for _, templateName := range []string{"deployment", "service", "certificate", "ingress-route"} {
+		templateFiles := []string{"deployment"}
+		if processType == "web" {
+			templateFiles = append(templateFiles, "service", "certificate", "ingress-route", "https-redirect-middleware")
+		}
+		for _, templateName := range templateFiles {
 			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
 			if err != nil {
 				return fmt.Errorf("Error reading %s template: %w", templateName, err)

From ee23dac2f372d4fb70725d26970966845a887007 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Tue, 23 Jan 2024 10:16:40 -0500
Subject: [PATCH 113/113] fix: respect the namespace

---
 plugins/scheduler-k3s/templates/chart/ingress-route.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
index 1061eb114..c4ccce5c9 100644
--- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
+++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml
@@ -28,7 +28,7 @@ spec:
       {{- if and $config.web.tls.enabled (and (eq $port_map.scheme "http") $port_map.redirect_to_https) }}
       middlewares:
         - name: {{ $.Values.global.app_name}}-redirect-to-https
-          namespace: traefik
+          namespace: {{ $.Values.global.namespace }}
       {{- end }}
       services:
       - name: {{ $.Values.global.app_name }}-{{ $processName }}