feat: support per-app letsencrypt emails on k3s

The `letsencrypt-email-prod` and `letsencrypt-email-stag` properties can now be set per app in addition to globally, resolving app-level before the global value for the app's selected `letsencrypt-server`. An app that sets its own email renders a namespaced cert-manager `Issuer` using that email, while apps without an override continue to use the shared `ClusterIssuer` with the global email.

plugins/scheduler-k3s/certificate_template_test.go

@@ -0,0 +1,199 @@
+package scheduler_k3s
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+)
+
+// renderCertificateChart renders the certificate.yaml and issuer.yaml templates
+// together and returns the parsed documents for each, keyed by template file name.
+func renderCertificateChart(t *testing.T, values map[string]interface{}) map[string][]map[string]interface{} {
+	t.Helper()
+
+	chartDir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n")
+	if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil {
+		t.Fatalf("write Chart.yaml: %v", err)
+	}
+
+	for _, name := range []string{"certificate", "issuer", "_helpers"} {
+		ext := ".yaml"
+		if name == "_helpers" {
+			ext = ".tpl"
+		}
+		tpl, err := templates.ReadFile("templates/chart/" + name + ext)
+		if err != nil {
+			t.Fatalf("read %s template: %v", name, err)
+		}
+		if err := os.WriteFile(filepath.Join(chartDir, "templates", name+ext), tpl, 0o644); err != nil {
+			t.Fatalf("write %s template: %v", name, err)
+		}
+	}
+
+	loaded, err := loader.Load(chartDir)
+	if err != nil {
+		t.Fatalf("load chart: %v", err)
+	}
+
+	renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil)
+	if err != nil {
+		t.Fatalf("ToRenderValues: %v", err)
+	}
+
+	rendered, err := engine.Render(loaded, renderValues)
+	if err != nil {
+		t.Fatalf("render: %v", err)
+	}
+
+	result := map[string][]map[string]interface{}{}
+	for _, name := range []string{"certificate", "issuer"} {
+		manifest := rendered["test/templates/"+name+".yaml"]
+		var docs []map[string]interface{}
+		decoder := yaml.NewDecoder(strings.NewReader(manifest))
+		for {
+			var doc map[string]interface{}
+			if err := decoder.Decode(&doc); err != nil {
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				t.Fatalf("yaml decode of %s failed: %v\nrendered:\n%s", name, err, manifest)
+			}
+			if doc != nil {
+				docs = append(docs, doc)
+			}
+		}
+		result[name] = docs
+	}
+
+	return result
+}
+
+func testCertificateValues(issuerKind string, issuerName string, issuer map[string]interface{}) map[string]interface{} {
+	global := map[string]interface{}{
+		"app_name":  "myapp",
+		"namespace": "myns",
+	}
+	if issuer != nil {
+		global["issuer"] = issuer
+	}
+
+	web := map[string]interface{}{
+		"domains": []interface{}{
+			map[string]interface{}{"name": "app.example.com"},
+		},
+		"tls": map[string]interface{}{
+			"enabled":           true,
+			"use_imported_cert": false,
+			"issuer_kind":       issuerKind,
+			"issuer_name":       issuerName,
+		},
+	}
+
+	return map[string]interface{}{
+		"global": global,
+		"processes": map[string]interface{}{
+			"web": map[string]interface{}{
+				"web": web,
+			},
+		},
+	}
+}
+
+func certificateIssuerRef(t *testing.T, docs map[string][]map[string]interface{}) map[string]interface{} {
+	t.Helper()
+	certs := docs["certificate"]
+	if len(certs) != 1 {
+		t.Fatalf("expected 1 certificate document, got %d: %#v", len(certs), certs)
+	}
+	spec, ok := certs[0]["spec"].(map[string]interface{})
+	if !ok {
+		t.Fatalf("expected certificate spec, got %#v", certs[0]["spec"])
+	}
+	issuerRef, ok := spec["issuerRef"].(map[string]interface{})
+	if !ok {
+		t.Fatalf("expected certificate issuerRef, got %#v", spec["issuerRef"])
+	}
+	return issuerRef
+}
+
+func TestCertificateTemplateUsesSharedClusterIssuerByDefault(t *testing.T) {
+	docs := renderCertificateChart(t, testCertificateValues("ClusterIssuer", "letsencrypt-prod", map[string]interface{}{"enabled": false}))
+
+	issuerRef := certificateIssuerRef(t, docs)
+	if issuerRef["kind"] != "ClusterIssuer" {
+		t.Fatalf("expected issuerRef.kind ClusterIssuer, got %#v", issuerRef["kind"])
+	}
+	if issuerRef["name"] != "letsencrypt-prod" {
+		t.Fatalf("expected issuerRef.name letsencrypt-prod, got %#v", issuerRef["name"])
+	}
+
+	if len(docs["issuer"]) != 0 {
+		t.Fatalf("expected no Issuer document for shared ClusterIssuer, got %#v", docs["issuer"])
+	}
+}
+
+func TestCertificateTemplateDefaultsKindWhenIssuerKindEmpty(t *testing.T) {
+	docs := renderCertificateChart(t, testCertificateValues("", "letsencrypt-prod", map[string]interface{}{"enabled": false}))
+
+	issuerRef := certificateIssuerRef(t, docs)
+	if issuerRef["kind"] != "ClusterIssuer" {
+		t.Fatalf("expected empty issuer_kind to default to ClusterIssuer, got %#v", issuerRef["kind"])
+	}
+}
+
+func TestCertificateTemplateRendersPerAppNamespacedIssuer(t *testing.T) {
+	issuer := map[string]interface{}{
+		"enabled":       true,
+		"name":          "myapp-letsencrypt-stag",
+		"email":         "app@dokku.me",
+		"server":        LetsencryptServerStag,
+		"ingress_class": "nginx",
+	}
+	docs := renderCertificateChart(t, testCertificateValues("Issuer", "myapp-letsencrypt-stag", issuer))
+
+	issuerRef := certificateIssuerRef(t, docs)
+	if issuerRef["kind"] != "Issuer" {
+		t.Fatalf("expected issuerRef.kind Issuer, got %#v", issuerRef["kind"])
+	}
+	if issuerRef["name"] != "myapp-letsencrypt-stag" {
+		t.Fatalf("expected issuerRef.name myapp-letsencrypt-stag, got %#v", issuerRef["name"])
+	}
+
+	issuers := docs["issuer"]
+	if len(issuers) != 1 {
+		t.Fatalf("expected 1 Issuer document, got %d: %#v", len(issuers), issuers)
+	}
+	if issuers[0]["kind"] != "Issuer" {
+		t.Fatalf("expected kind Issuer, got %#v", issuers[0]["kind"])
+	}
+
+	metadata, _ := issuers[0]["metadata"].(map[string]interface{})
+	if metadata["name"] != "myapp-letsencrypt-stag" {
+		t.Fatalf("expected Issuer name myapp-letsencrypt-stag, got %#v", metadata["name"])
+	}
+	if metadata["namespace"] != "myns" {
+		t.Fatalf("expected Issuer namespace myns, got %#v", metadata["namespace"])
+	}
+
+	spec, _ := issuers[0]["spec"].(map[string]interface{})
+	acme, _ := spec["acme"].(map[string]interface{})
+	if acme["email"] != "app@dokku.me" {
+		t.Fatalf("expected Issuer acme email app@dokku.me, got %#v", acme["email"])
+	}
+	if acme["server"] != LetsencryptServerStag {
+		t.Fatalf("expected Issuer acme server %q, got %#v", LetsencryptServerStag, acme["server"])
+	}
+}

plugins/scheduler-k3s/chart.go

@@ -119,24 +119,26 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 	}
 
 	tlsEnabled := false
+	issuerKind := ""
 	issuerName := ""
 	useImportedCert := false
+	appIssuer := AppIssuer{}
 
 	if importedCertExists {
 		tlsEnabled = true
 		useImportedCert = true
 	} else {
 		server := getComputedLetsencryptServer(appName)
-		letsencryptEmailStag := getGlobalLetsencryptEmailStag()
-		letsencryptEmailProd := getGlobalLetsencryptEmailProd()
 
 		switch server {
 		case "prod", "production":
-			issuerName = "letsencrypt-prod"
-			tlsEnabled = letsencryptEmailProd != ""
+			computedEmail := getComputedLetsencryptEmailProd(appName)
+			tlsEnabled = computedEmail != ""
+			issuerKind, issuerName, appIssuer = resolveLetsencryptIssuer(appName, "letsencrypt-prod", getLetsencryptEmailProd(appName), computedEmail, LetsencryptServerProd)
 		case "stag", "staging":
-			issuerName = "letsencrypt-stag"
-			tlsEnabled = letsencryptEmailStag != ""
+			computedEmail := getComputedLetsencryptEmailStag(appName)
+			tlsEnabled = computedEmail != ""
+			issuerKind, issuerName, appIssuer = resolveLetsencryptIssuer(appName, "letsencrypt-stag", getLetsencryptEmailStag(appName), computedEmail, LetsencryptServerStag)
 		case "false":
 			issuerName = ""
 			tlsEnabled = false
@@ -297,6 +299,7 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 				Type:             imageSourceType,
 				WorkingDir:       workingDir,
 			},
+			Issuer:    appIssuer,
 			Labels:    globalLabels,
 			Namespace: namespace,
 			Network: GlobalNetwork{
@@ -421,6 +424,7 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 				PortMaps: []ProcessPortMap{},
 				TLS: ProcessTls{
 					Enabled:         tlsEnabled,
+					IssuerKind:      issuerKind,
 					IssuerName:      issuerName,
 					UseImportedCert: useImportedCert,
 				},
@@ -489,7 +493,7 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 
 		templateFiles := []string{"deployment", "keda-scaled-object"}
 		if processType == "web" {
-			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "compression-middleware", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
+			templateFiles = append(templateFiles, "service", "certificate", "issuer", "ingress", "ingress-route", "compression-middleware", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
 		}
 		for _, templateName := range templateFiles {
 			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))

plugins/scheduler-k3s/functions.go

@@ -309,14 +309,14 @@ func applyClusterIssuers(ctx context.Context) error {
 				Enabled:      letsencryptEmailStag != "",
 				IngressClass: getComputedIngressClass(),
 				Name:         "letsencrypt-stag",
-				Server:       "https://acme-staging-v02.api.letsencrypt.org/directory",
+				Server:       LetsencryptServerStag,
 			},
 			"letsencrypt-prod": {
 				Email:        letsencryptEmailProd,
 				Enabled:      letsencryptEmailProd != "",
 				IngressClass: getComputedIngressClass(),
 				Name:         "letsencrypt-prod",
-				Server:       "https://acme-v02.api.letsencrypt.org/directory",
+				Server:       LetsencryptServerProd,
 			},
 		},
 	}
@@ -1233,20 +1233,57 @@ func getComputedLetsencryptServer(appName string) string {
 	return letsencryptServer
 }
 
+func getLetsencryptEmailProd(appName string) string {
+	return common.PropertyGet("scheduler-k3s", appName, "letsencrypt-email-prod")
+}
+
 func getGlobalLetsencryptEmailProd() string {
 	return common.PropertyGet("scheduler-k3s", "--global", "letsencrypt-email-prod")
 }
 
-func getComputedLetsencryptEmailProd() string {
-	return getGlobalLetsencryptEmailProd()
+func getComputedLetsencryptEmailProd(appName string) string {
+	letsencryptEmail := getLetsencryptEmailProd(appName)
+	if letsencryptEmail == "" {
+		letsencryptEmail = getGlobalLetsencryptEmailProd()
+	}
+
+	return letsencryptEmail
+}
+
+func getLetsencryptEmailStag(appName string) string {
+	return common.PropertyGet("scheduler-k3s", appName, "letsencrypt-email-stag")
 }
 
 func getGlobalLetsencryptEmailStag() string {
 	return common.PropertyGet("scheduler-k3s", "--global", "letsencrypt-email-stag")
 }
 
-func getComputedLetsencryptEmailStag() string {
-	return getGlobalLetsencryptEmailStag()
+func getComputedLetsencryptEmailStag(appName string) string {
+	letsencryptEmail := getLetsencryptEmailStag(appName)
+	if letsencryptEmail == "" {
+		letsencryptEmail = getGlobalLetsencryptEmailStag()
+	}
+
+	return letsencryptEmail
+}
+
+// resolveLetsencryptIssuer determines the issuer kind and name an app's Certificate should
+// reference for the selected letsencrypt server. When the app sets its own email for that
+// server, a namespaced Issuer is rendered into the app's chart using the app's email;
+// otherwise the app references the shared global ClusterIssuer.
+func resolveLetsencryptIssuer(appName string, clusterIssuerName string, appEmail string, computedEmail string, server string) (string, string, AppIssuer) {
+	if appEmail == "" {
+		return "ClusterIssuer", clusterIssuerName, AppIssuer{}
+	}
+
+	issuerName := fmt.Sprintf("%s-%s", appName, clusterIssuerName)
+	return "Issuer", issuerName, AppIssuer{
+		Email:        computedEmail,
+		Enabled:      true,
+		IngressClass: getComputedIngressClass(),
+		Name:         issuerName,
+		Server:       server,
+	}
 }
 
 func getKustomizeDirectory(appName string) string {

plugins/scheduler-k3s/report.go

@@ -89,8 +89,10 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 			"--scheduler-k3s-letsencrypt-server":              reportLetsencryptServer,
 			"--scheduler-k3s-global-letsencrypt-server":       reportGlobalLetsencryptServer,
 			"--scheduler-k3s-computed-letsencrypt-email-prod": reportComputedLetsencryptEmailProd,
+			"--scheduler-k3s-letsencrypt-email-prod":          reportLetsencryptEmailProd,
 			"--scheduler-k3s-global-letsencrypt-email-prod":   reportGlobalLetsencryptEmailProd,
 			"--scheduler-k3s-computed-letsencrypt-email-stag": reportComputedLetsencryptEmailStag,
+			"--scheduler-k3s-letsencrypt-email-stag":          reportLetsencryptEmailStag,
 			"--scheduler-k3s-global-letsencrypt-email-stag":   reportGlobalLetsencryptEmailStag,
 			"--scheduler-k3s-computed-namespace":              reportComputedNamespace,
 			"--scheduler-k3s-namespace":                       reportNamespace,
@@ -658,7 +660,11 @@ func reportGlobalLetsencryptServer(appName string) string {
 }
 
 func reportComputedLetsencryptEmailProd(appName string) string {
-	return getComputedLetsencryptEmailProd()
+	return getComputedLetsencryptEmailProd(appName)
+}
+
+func reportLetsencryptEmailProd(appName string) string {
+	return getLetsencryptEmailProd(appName)
 }
 
 func reportGlobalLetsencryptEmailProd(appName string) string {
@@ -666,7 +672,11 @@ func reportGlobalLetsencryptEmailProd(appName string) string {
 }
 
 func reportComputedLetsencryptEmailStag(appName string) string {
-	return getComputedLetsencryptEmailStag()
+	return getComputedLetsencryptEmailStag(appName)
+}
+
+func reportLetsencryptEmailStag(appName string) string {
+	return getLetsencryptEmailStag(appName)
 }
 
 func reportGlobalLetsencryptEmailStag(appName string) string {

plugins/scheduler-k3s/scheduler_k3s.go

@@ -18,13 +18,15 @@ import (
 var (
 	// DefaultProperties is a map of all valid k3s properties with corresponding default property values
 	DefaultProperties = map[string]string{
-		"deploy-timeout":      "",
-		"letsencrypt-server":  "",
-		"kustomize-root-path": "",
-		"image-pull-secrets":  "",
-		"namespace":           "",
-		"rollback-on-failure": "",
-		"shm-size":            "",
+		"deploy-timeout":         "",
+		"letsencrypt-email-prod": "",
+		"letsencrypt-email-stag": "",
+		"letsencrypt-server":     "",
+		"kustomize-root-path":    "",
+		"image-pull-secrets":     "",
+		"namespace":              "",
+		"rollback-on-failure":    "",
+		"shm-size":               "",
 	}
 
 	// GlobalProperties is a map of all valid global k3s properties
@@ -51,6 +53,8 @@ const GlobalProcessType = "--global"
 const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
 const DefaultKubeContext = ""
 const TriggerAuthPropertyPrefix = "trigger-auth."
+const LetsencryptServerProd = "https://acme-v02.api.letsencrypt.org/directory"
+const LetsencryptServerStag = "https://acme-staging-v02.api.letsencrypt.org/directory"
 
 // AnnotationResourceTypes lists the kubernetes resource types that scheduler-k3s
 // supports user-provided annotations for. The order here is also the iteration order

plugins/scheduler-k3s/template.go

@@ -46,6 +46,7 @@ type GlobalValues struct {
 	AppName         string             `yaml:"app_name"`
 	DeploymentID    string             `yaml:"deployment_id"`
 	Image           GlobalImage        `yaml:"image"`
+	Issuer          AppIssuer          `yaml:"issuer,omitempty"`
 	Labels          ProcessLabels      `yaml:"labels,omitempty"`
 	Keda            GlobalKedaValues   `yaml:"keda"`
 	Namespace       string             `yaml:"namespace"`
@@ -343,10 +344,19 @@ func (a NameSorter) Less(i, j int) bool { return a[i].Name < a[j].Name }
 
 type ProcessTls struct {
 	Enabled         bool   `yaml:"enabled"`
+	IssuerKind      string `yaml:"issuer_kind"`
 	IssuerName      string `yaml:"issuer_name"`
 	UseImportedCert bool   `yaml:"use_imported_cert"`
 }
 
+type AppIssuer struct {
+	Email        string `yaml:"email"`
+	Enabled      bool   `yaml:"enabled"`
+	IngressClass string `yaml:"ingress_class"`
+	Name         string `yaml:"name"`
+	Server       string `yaml:"server"`
+}
+
 type ClusterIssuer struct {
 	Email        string `yaml:"email"`
 	Enabled      bool   `yaml:"enabled"`

plugins/scheduler-k3s/templates/chart/certificate.yaml

@@ -22,7 +22,7 @@ metadata:
   namespace: {{ $.Values.global.namespace }}
 spec:
   issuerRef:
-    kind: ClusterIssuer
+    kind: {{ $config.web.tls.issuer_kind | default "ClusterIssuer" }}
     name: {{ $config.web.tls.issuer_name }}
   secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
   secretTemplate:

plugins/scheduler-k3s/templates/chart/issuer.yaml

@@ -0,0 +1,23 @@
+{{- if .Values.global.issuer.enabled }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+  labels:
+    app.kubernetes.io/name: {{ .Values.global.issuer.name }}
+    app.kubernetes.io/part-of: {{ .Values.global.app_name }}
+  name: {{ .Values.global.issuer.name }}
+  namespace: {{ .Values.global.namespace }}
+spec:
+  acme:
+    email: {{ .Values.global.issuer.email }}
+    server: {{ .Values.global.issuer.server }}
+    privateKeySecretRef:
+      name: {{ .Values.global.issuer.name }}
+    solvers:
+    - http01:
+        ingress:
+          class: {{ .Values.global.issuer.ingress_class }}
+{{- end }}