diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index a85c189a6..d3ff4f6a0 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -271,9 +271,14 @@ The default value may be set by passing an empty value for the option. dokku scheduler-k3s:set --global letsencrypt-server staging ``` -### Customizing Resource Annotations +### Customizing Annotations and Labels -Dokku injects certain resources into each created resource by default, but it may be necessary to inject others for tighter integration with third-party tools. The `scheduler-k3s:annotations:set` command can be used to perform this task. The command takes an app name and a required `--resource-type` flag. +> [!NOTE] +> The cron ID is used as the process type if your app deploys any cron tasks + +#### Setting Annotations + +Dokku injects certain annotations into each created resource by default, but it may be necessary to inject others for tighter integration with third-party tools. The `scheduler-k3s:annotations:set` command can be used to perform this task. The command takes an app name and a required `--resource-type` flag. ```shell dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value --resource-type deployment @@ -281,20 +286,10 @@ dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value If not specified, the annotation will be applied to all processes within an app, though it may be further scoped to a specific process type via the `--process-type` flag. -> [!NOTE] -> The cron ID is used as the process type if your app deploys any cron tasks - ```shell dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value --resource-type deployment --process-type web ``` -To unset an annotation, pass an empty value: - -```shell -dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment -dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment --process-type web -``` - The following resource types are supported: - `certificate` @@ -309,6 +304,52 @@ The following resource types are supported: - `traefik_ingressroute` - `traefik_middleware` +#### Removing an annotation + +To unset an annotation, pass an empty value: + +```shell +dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment +dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment --process-type web +``` + +#### Setting Labels + +Dokku injects certain labels into each created resource by default, but it may be necessary to inject others for tighter integration with third-party tools. The `scheduler-k3s:labels:set` command can be used to perform this task. The command takes an app name and a required `--resource-type` flag. + +```shell +dokku scheduler-k3s:labels:set node-js-app label.key label.value --resource-type deployment +``` + +If not specified, the label will be applied to all processes within an app, though it may be further scoped to a specific process type via the `--process-type` flag. + +```shell +dokku scheduler-k3s:labels:set node-js-app label.key label.value --resource-type deployment --process-type web +``` + +The following resource types are supported: + +- `certificate` +- `cronjob` +- `deployment` +- `ingress` +- `job` +- `pod` +- `secret` +- `service` +- `serviceaccount` +- `traefik_ingressroute` +- `traefik_middleware` + +#### Removing a label + +To unset an label, pass an empty value: + +```shell +dokku scheduler-k3s:annotations:set node-js-app label.key --resource-type deployment +dokku scheduler-k3s:labels:set node-js-app label.key --resource-type deployment --process-type web +``` + ### Using kubectl remotely > [!WARNING] diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index 50a1184af..c150d6aa1 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,4 +1,4 @@ -SUBCOMMANDS = subcommands/initialize subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall +SUBCOMMANDS = subcommands/annotations:set subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 0c32b281f..3b18c0540 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -675,6 +675,103 @@ func getIngressAnnotations(appName string, processType string) (map[string]strin return annotations, nil } +// getLabels retrieves labels for a given app and process type +func getLabels(appName string, processType string) (ProcessLabels, error) { + labels := ProcessLabels{} + certificateLabels, err := getLabel(appName, processType, "certificate") + if err != nil { + return labels, err + } + labels.CertificateLabels = certificateLabels + + cronJobLabels, err := getLabel(appName, processType, "cronjob") + if err != nil { + return labels, err + } + labels.CronJobLabels = cronJobLabels + + deploymentLabels, err := getLabel(appName, processType, "deployment") + if err != nil { + return labels, err + } + labels.DeploymentLabels = deploymentLabels + + ingressLabels, err := getLabel(appName, processType, "ingress") + if err != nil { + return labels, err + } + labels.IngressLabels = ingressLabels + + jobLabels, err := getLabel(appName, processType, "job") + if err != nil { + return labels, err + } + labels.JobLabels = jobLabels + + podLabels, err := getLabel(appName, processType, "pod") + if err != nil { + return labels, err + } + labels.PodLabels = podLabels + + secretLabels, err := getLabel(appName, processType, "secret") + if err != nil { + return labels, err + } + labels.SecretLabels = secretLabels + + serviceLabels, err := getLabel(appName, processType, "service") + if err != nil { + return labels, err + } + labels.ServiceLabels = serviceLabels + + serviceAccountLabels, err := getLabel(appName, processType, "serviceaccount") + if err != nil { + return labels, err + } + labels.ServiceAccountLabels = serviceAccountLabels + + traefikIngressRouteLabels, err := getLabel(appName, processType, "traefik_ingressroute") + if err != nil { + return labels, err + } + labels.TraefikIngressRouteLabels = traefikIngressRouteLabels + + traefikMiddlewareLabels, err := getLabel(appName, processType, "traefik_middleware") + if err != nil { + return labels, err + } + labels.TraefikMiddlewareLabels = traefikMiddlewareLabels + + return labels, nil +} + +// getGlobalLabel retrieves global labels for a given app +func getGlobalLabel(appName string) (ProcessLabels, error) { + return getLabels(appName, GlobalProcessType) +} + +// getLabel retrieves an label for a given app, process type, and resource type +func getLabel(appName string, processType string, resourceType string) (map[string]string, error) { + labels := map[string]string{} + labelsList, err := common.PropertyListGet("scheduler-k3s", appName, fmt.Sprintf("labels.%s.%s", processType, resourceType)) + if err != nil { + return labels, err + } + + for _, label := range labelsList { + parts := strings.SplitN(label, ": ", 2) + if len(parts) != 2 { + return labels, fmt.Errorf("Invalid label format: %s", label) + } + + labels[parts[0]] = parts[1] + } + + return labels, nil +} + func getLetsencryptServer(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "") } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 62ce97ed4..39ce66f11 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -150,7 +150,7 @@ var WorkerLabels = map[string]string{ "node-role.kubernetes.io/worker": "worker", } -//go:embed templates/* +//go:embed all:templates var templates embed.FS func init() { diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index a1fd3b67b..6eb64640a 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -7,9 +7,6 @@ import ( "strings" "github.com/dokku/dokku/plugins/common" - scheduler_k3s "github.com/dokku/dokku/plugins/scheduler-k3s" - - flag "github.com/spf13/pflag" ) const ( @@ -50,28 +47,6 @@ func main() { } else { fmt.Print("\n scheduler-k3s, Manage scheduler-k3s settings for an app\n") } - case "scheduler-k3s:annotations:set": - args := flag.NewFlagSet("scheduler-k3s:annotations:set", flag.ExitOnError) - global := args.Bool("global", false, "--global: set a global property") - processType := args.String("process-type", "", "--process-type: scope to process-type") - resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type") - err := args.Parse(os.Args[2:]) - if err != nil { - common.LogFailWithError(err) - } - - appName := args.Arg(0) - property := args.Arg(1) - value := args.Arg(2) - if *global { - appName = "--global" - property = args.Arg(0) - value = args.Arg(1) - } - - if err := scheduler_k3s.CommandAnnotationsSet(appName, *processType, *resourceType, property, value); err != nil { - common.LogFailWithError(err) - } default: dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT")) if err != nil { diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index cf45bf252..4685d0834 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -18,13 +18,22 @@ func main() { var err error switch subcommand { - case "initialize": - args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) - taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") - serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node") - ingressClass := args.String("ingress-class", "traefik", "ingress-class: ingress-class to use for all outbound traffic") + case "annotations:set": + args := flag.NewFlagSet("scheduler-k3s:annotations:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: set a global property") + processType := args.String("process-type", "", "--process-type: scope to process-type") + resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling) + appName := args.Arg(0) + property := args.Arg(1) + value := args.Arg(2) + if *global { + appName = "--global" + property = args.Arg(0) + value = args.Arg(1) + } + + err = scheduler_k3s.CommandAnnotationsSet(appName, *processType, *resourceType, property, value) case "cluster-add": args := flag.NewFlagSet("scheduler-k3s:cluster-add", flag.ExitOnError) allowUknownHosts := args.Bool("insecure-allow-unknown-hosts", false, "insecure-allow-unknown-hosts: allow unknown hosts") @@ -44,6 +53,29 @@ func main() { args.Parse(os.Args[2:]) nodeName := args.Arg(0) err = scheduler_k3s.CommandClusterRemove(nodeName) + case "initialize": + args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) + taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") + serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node") + ingressClass := args.String("ingress-class", "traefik", "ingress-class: ingress-class to use for all outbound traffic") + args.Parse(os.Args[2:]) + err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling) + case "labels:set": + args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: set a global property") + processType := args.String("process-type", "", "--process-type: scope to process-type") + resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + property := args.Arg(1) + value := args.Arg(2) + if *global { + appName = "--global" + property = args.Arg(0) + value = args.Arg(1) + } + + err = scheduler_k3s.CommandLabelsSet(appName, *processType, *resourceType, property, value) case "report": args := flag.NewFlagSet("scheduler-k3s:report", flag.ExitOnError) format := args.String("format", "stdout", "format: [ stdout | json ]") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index a01b53265..681e426ae 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -713,6 +713,47 @@ func CommandClusterRemove(nodeName string) error { return nil } +// CommandLabelsSet set or clear a scheduler-k3s label for an app +func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error { + if resourceType == "" { + return fmt.Errorf("Missing resource-type") + } + + if processType == "" { + processType = GlobalProcessType + } + + property := fmt.Sprintf("labels.%s.%s", processType, resourceType) + labelsList, err := common.PropertyListGet("scheduler-k3s", appName, property) + if err != nil { + return fmt.Errorf("Unable to get property list: %w", err) + } + + labels := []string{} + for _, annotation := range labelsList { + parts := strings.SplitN(annotation, ": ", 2) + if len(parts) != 2 { + return fmt.Errorf("Invalid annotation: %s", annotation) + } + if key == parts[0] { + continue + } + + labels = append(labels, annotation) + } + + if value != "" { + labels = append(labels, fmt.Sprintf("%s: %s", key, value)) + } + + sort.Strings(labels) + if err := common.PropertyListWrite("scheduler-k3s", appName, property, labels); err != nil { + return fmt.Errorf("Unable to write property list: %w", err) + } + + return nil +} + // CommandReport displays a scheduler-k3s report for one or more apps func CommandReport(appName string, format string, infoFlag string) error { if len(appName) == 0 { diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 5003a3965..1af9c8445 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -37,6 +37,7 @@ type GlobalValues struct { AppName string `yaml:"app_name"` DeploymentID string `yaml:"deploment_id"` Image GlobalImage `yaml:"image"` + Labels ProcessLabels `yaml:"labels,omitempty"` Namespace string `yaml:"namespace"` Network GlobalNetwork `yaml:"network"` Secrets map[string]string `yaml:"secrets,omitempty"` @@ -60,6 +61,7 @@ type ProcessValues struct { Args []string `yaml:"args,omitempty"` Cron ProcessCron `yaml:"cron,omitempty"` Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"` + Labels ProcessLabels `yaml:"labels,omitempty"` ProcessType ProcessType `yaml:"process_type"` Replicas int32 `yaml:"replicas"` Resources ProcessResourcesMap `yaml:"resources,omitempty"` @@ -129,6 +131,20 @@ const ( URISchemeHTTPS URIScheme = "HTTPS" ) +type ProcessLabels struct { + CertificateLabels map[string]string `yaml:"certificate,omitempty"` + CronJobLabels map[string]string `yaml:"cronjob,omitempty"` + DeploymentLabels map[string]string `yaml:"deployment,omitempty"` + IngressLabels map[string]string `yaml:"ingress,omitempty"` + JobLabels map[string]string `yaml:"job,omitempty"` + PodLabels map[string]string `yaml:"pod,omitempty"` + SecretLabels map[string]string `yaml:"secret,omitempty"` + ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` + ServiceLabels map[string]string `yaml:"service,omitempty"` + TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` + TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` +} + type ProcessWeb struct { Domains []string `yaml:"domains,omitempty"` PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"` diff --git a/plugins/scheduler-k3s/templates/chart/_helpers.tpl b/plugins/scheduler-k3s/templates/chart/_helpers.tpl new file mode 100644 index 000000000..3f340c56c --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/_helpers.tpl @@ -0,0 +1,17 @@ +{{- define "print.annotations" }} +{{- if and .config.annotations (hasKey .config.annotations .key) }} +{{- $annotations := get .config.annotations .key }} +{{- range $k, $v := $annotations }} +{{ $k }}: {{ $v | quote }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "print.labels" }} +{{- if and .config.labels (hasKey .config.labels .key) }} +{{- $labels := get .config.labels .key }} +{{- range $k, $v := $labels }} +{{ $k }}: {{ $v | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml index 0ee993927..1ffc42a92 100644 --- a/plugins/scheduler-k3s/templates/chart/certificate.yaml +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -7,19 +7,13 @@ kind: Certificate metadata: annotations: dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.certificate }} - {{- range $k, $v := $.Values.global.annotations.certificate }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.certificate }} - {{- range $k, $v := $config.annotations.certificate }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "certificate") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "certificate") | indent 4 }} labels: app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "certificate") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "certificate") | indent 4 }} name: {{ $.Values.global.app_name }}-{{ $processName }} namespace: {{ $.Values.global.namespace }} spec: @@ -33,6 +27,8 @@ spec: labels: app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 6 }} + {{ include "print.labels" (dict "config" $config "key" "secret") | indent 6 }} dnsNames: {{- range $config.web.domains }} - {{ . }} diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index ddd5cbefb..9363322f7 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -11,21 +11,15 @@ metadata: dokku.com/job-suffix: {{ $config.cron.suffix }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron - {{- if and $.Values.global.annotations $.Values.global.annotations.cronjob }} - {{- range $k, $v := $.Values.global.annotations.cronjob }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.cronjob }} - {{- range $k, $v := $config.annotations.cronjob }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "cronjob") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "cronjob") | indent 4 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} app.kubernetes.io/name: cron app.kubernetes.io/part-of: {{ $.Values.global.app_name }} dokku.com/cron-id: {{ $config.cron.id }} + {{ include "print.labels" (dict "config" $.Values.global "key" "cronjob") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "cronjob") | indent 4 }} name: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} namespace: {{ $.Values.global.namespace }} spec: @@ -40,21 +34,15 @@ spec: dokku.com/job-suffix: {{ $config.cron.suffix }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron - {{- if and $.Values.global.annotations $.Values.global.annotations.pod }} - {{- range $k, $v := $.Values.global.annotations.pod }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.pod }} - {{- range $k, $v := $config.annotations.pod }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "job") | indent 8 }} + {{ include "print.annotations" (dict "config" $config "key" "job") | indent 8 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} app.kubernetes.io/name: cron app.kubernetes.io/part-of: {{ $.Values.global.app_name }} dokku.com/cron-id: {{ $config.cron.id }} + {{ include "print.labels" (dict "config" $.Values.global "key" "job") | indent 8 }} + {{ include "print.labels" (dict "config" $config "key" "job") | indent 8 }} spec: backoffLimit: 0 podReplacementPolicy: Failed @@ -68,11 +56,15 @@ spec: dokku.com/job-suffix: {{ $config.cron.suffix }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + {{ include "print.annotations" (dict "config" $.Values.global "key" "pod") | indent 12 }} + {{ include "print.annotations" (dict "config" $config "key" "pod") | indent 12 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} app.kubernetes.io/name: cron app.kubernetes.io/part-of: {{ $.Values.global.app_name }} dokku.com/cron-id: {{ $config.cron.id }} + {{ include "print.labels" (dict "config" $.Values.global "key" "pod") | indent 12 }} + {{ include "print.labels" (dict "config" $config "key" "pod") | indent 12 }} spec: containers: - args: diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index b2688edc9..40f32c6af 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -15,20 +15,14 @@ metadata: dokku.com/builder-type: {{ $.Values.global.image.type }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }} - {{- if and $.Values.global.annotations $.Values.global.annotations.deployment }} - {{- range $k, $v := $.Values.global.annotations.deployment }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.deployment }} - {{- range $k, $v := $config.annotations.deployment }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "deployment") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "deployment") | indent 4 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "deployment") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "deployment") | indent 4 }} name: {{ $.Values.global.app_name }}-{{ $processName }} namespace: {{ $.Values.global.namespace }} spec: @@ -39,6 +33,8 @@ spec: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "pod") | indent 6 }} + {{ include "print.labels" (dict "config" $config "key" "pod") | indent 6 }} strategy: {} template: metadata: @@ -47,20 +43,14 @@ spec: dokku.com/builder-type: {{ $.Values.global.image.type }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }} - {{- if and $.Values.global.annotations $.Values.global.annotations.pod }} - {{- range $k, $v := $.Values.global.annotations.pod }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.pod }} - {{- range $k, $v := $config.annotations.pod }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "pod") | indent 8 }} + {{ include "print.annotations" (dict "config" $config "key" "pod") | indent 8 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "pod") | indent 8 }} + {{ include "print.labels" (dict "config" $config "key" "pod") | indent 8 }} spec: containers: - args: diff --git a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml index fef18ded1..20a3d4429 100644 --- a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml +++ b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml @@ -7,20 +7,14 @@ kind: Middleware metadata: annotations: dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.traefik_middleware }} - {{- range $k, $v := $.Values.global.annotations.traefik_middleware }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.traefik_middleware }} - {{- range $k, $v := $config.annotations.traefik_middleware }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "traefik_middleware") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "traefik_middleware") | indent 4 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}-redirect-to-https app.kubernetes.io/name: {{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "traefik_middleware") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "traefik_middleware") | indent 4 }} name: {{ $.Values.global.app_name}}-{{ $processName }}-redirect-to-https namespace: {{ $.Values.global.namespace }} spec: diff --git a/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml b/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml index 934c15886..ac5d44cbe 100644 --- a/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml +++ b/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml @@ -6,15 +6,12 @@ metadata: annotations: app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.secret }} - {{- range $k, $v := $.Values.global.annotations.secret }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "secret") | indent 4 }} labels: app.kubernetes.io/instance: ims-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} app.kubernetes.io/name: ims-{{ $.Values.global.app_name }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 4 }} name: ims-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} namespace: {{ $.Values.global.namespace }} data: diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index 578fa7b3c..e61c48063 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -15,20 +15,14 @@ kind: IngressRoute metadata: annotations: dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.traefik_ingressroute }} - {{- range $k, $v := $.Values.global.annotations.traefik_ingressroute }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.traefik_ingressroute }} - {{- range $k, $v := $config.annotations.traefik_ingressroute }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "traefik_ingressroute") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "traefik_ingressroute") | indent 4 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "traefik_ingressroute") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "traefik_ingressroute") | indent 4 }} name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }} namespace: {{ $.Values.global.namespace }} spec: diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index f78290cdd..cfbf519df 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -15,23 +15,17 @@ kind: Ingress metadata: annotations: dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.ingress }} - {{- range $k, $v := $.Values.global.annotations.ingress }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.ingress }} - {{- range $k, $v := $config.annotations.ingress }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} {{- if $config.web.tls.enabled }} nginx.ingress.kubernetes.io/force-ssl-redirect: "true" {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "ingress") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "ingress") | indent 4 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "ingress") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "ingress") | indent 4 }} name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }} namespace: {{ $.Values.global.namespace }} spec: diff --git a/plugins/scheduler-k3s/templates/chart/secret.yaml b/plugins/scheduler-k3s/templates/chart/secret.yaml index f8ac42d21..47dd3adee 100644 --- a/plugins/scheduler-k3s/templates/chart/secret.yaml +++ b/plugins/scheduler-k3s/templates/chart/secret.yaml @@ -5,15 +5,12 @@ metadata: annotations: app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.secret }} - {{- range $k, $v := $.Values.global.annotations.secret }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "secret") | indent 4 }} labels: app.kubernetes.io/instance: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} app.kubernetes.io/name: env-{{ $.Values.global.app_name }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 4 }} name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} namespace: {{ $.Values.global.namespace }} {{- with .Values.global.secrets }} diff --git a/plugins/scheduler-k3s/templates/chart/service-account.yaml b/plugins/scheduler-k3s/templates/chart/service-account.yaml index eaf142e84..55f669b45 100644 --- a/plugins/scheduler-k3s/templates/chart/service-account.yaml +++ b/plugins/scheduler-k3s/templates/chart/service-account.yaml @@ -3,14 +3,11 @@ kind: ServiceAccount metadata: annotations: dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.serviceaccount }} - {{- range $k, $v := $.Values.global.annotations.serviceaccount }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "serviceaccount") | indent 4 }} labels: app.kubernetes.io/instance: service-account app.kubernetes.io/name: service-account app.kubernetes.io/part-of: "{{ $.Values.global.app_name }}" + {{ include "print.labels" (dict "config" $.Values.global "key" "serviceaccount") | indent 4 }} name: "{{ $.Values.global.app_name }}" namespace: "{{ $.Values.global.namespace }}" diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml index d6b812762..bf5d3d34b 100644 --- a/plugins/scheduler-k3s/templates/chart/service.yaml +++ b/plugins/scheduler-k3s/templates/chart/service.yaml @@ -6,20 +6,14 @@ kind: Service metadata: annotations: dokku.com/managed: "true" - {{- if and $.Values.global.annotations $.Values.global.annotations.service }} - {{- range $k, $v := $.Values.global.annotations.service }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} - {{- if and $config.annotations $config.annotations.service }} - {{- range $k, $v := $config.annotations.service }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} + {{ include "print.annotations" (dict "config" $.Values.global "key" "service") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "service") | indent 4 }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "service") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "service") | indent 4 }} name: {{ $.Values.global.app_name }}-{{ $processName }} namespace: {{ $.Values.global.namespace }} spec: diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index e4c2d597e..24be9066e 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -261,6 +261,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting global annotations: %w", err) } + globalLabels, err := getGlobalLabel(appName) + if err != nil { + return fmt.Errorf("Error getting global labels: %w", err) + } + values := &AppValues{ Global: GlobalValues{ Annotations: globalAnnotations, @@ -273,6 +278,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Type: imageSourceType, WorkingDir: workingDir, }, + Labels: globalLabels, Namespace: namespace, Network: GlobalNetwork{ IngressClass: getGlobalIngressClass(), @@ -316,10 +322,16 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting process annotations: %w", err) } + labels, err := getLabels(appName, processType) + if err != nil { + return fmt.Errorf("Error getting process labels: %w", err) + } + processValues := ProcessValues{ Annotations: annotations, Args: args, Healthchecks: processHealthchecks, + Labels: labels, ProcessType: ProcessType_Worker, Replicas: int32(processCount), Resources: processResources, @@ -458,6 +470,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting process annotations: %w", err) } + labels, err := getLabels(appName, cronEntry.ID) + if err != nil { + return fmt.Errorf("Error getting process labels: %w", err) + } + processValues := ProcessValues{ Args: words, Annotations: annotations, @@ -466,6 +483,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Schedule: cronEntry.Schedule, Suffix: suffix, }, + Labels: labels, ProcessType: ProcessType_Cron, Replicas: 1, Resources: processResources, @@ -493,6 +511,21 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e values.Global.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value)) } + b, err := templates.ReadFile("templates/chart/_helpers.tpl") + if err != nil { + return fmt.Errorf("Error reading _helpers template: %w", err) + } + + helpersFile := filepath.Join(chartDir, "templates", "_helpers.tpl") + err = os.WriteFile(helpersFile, b, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("Error writing _helpers template: %w", err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(helpersFile) + } + err = writeYaml(WriteYamlInput{ Object: values, Path: filepath.Join(chartDir, "values.yaml"), diff --git a/tests/unit/scheduler-k3s.bats b/tests/unit/scheduler-k3s.bats index c78e1f785..530d29bda 100644 --- a/tests/unit/scheduler-k3s.bats +++ b/tests/unit/scheduler-k3s.bats @@ -244,6 +244,11 @@ uninstall_k3s() { echo "status: $status" assert_success + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type job test.dokku.com/resource-type job" + echo "output: $output" + echo "status: $status" + assert_success + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment test.dokku.com/resource-type deployment" echo "output: $output" echo "status: $status" @@ -288,7 +293,7 @@ uninstall_k3s() { echo "output: $output" echo "status: $status" assert_success - assert_output "pod" + assert_output "job" run /bin/bash -c "kubectl get secret -o json | jq -r '.items[0].metadata.annotations.\"test.dokku.com/resource-type\"'" echo "output: $output"