diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index a85c189a6..d3ff4f6a0 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -271,9 +271,14 @@ The default value may be set by passing an empty value for the option. dokku scheduler-k3s:set --global letsencrypt-server staging ``` -### Customizing Resource Annotations +### Customizing Annotations and Labels -Dokku injects certain resources into each created resource by default, but it may be necessary to inject others for tighter integration with third-party tools. The `scheduler-k3s:annotations:set` command can be used to perform this task. The command takes an app name and a required `--resource-type` flag. +> [!NOTE] +> The cron ID is used as the process type if your app deploys any cron tasks + +#### Setting Annotations + +Dokku injects certain annotations into each created resource by default, but it may be necessary to inject others for tighter integration with third-party tools. The `scheduler-k3s:annotations:set` command can be used to perform this task. The command takes an app name and a required `--resource-type` flag. ```shell dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value --resource-type deployment @@ -281,20 +286,10 @@ dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value If not specified, the annotation will be applied to all processes within an app, though it may be further scoped to a specific process type via the `--process-type` flag. -> [!NOTE] -> The cron ID is used as the process type if your app deploys any cron tasks - ```shell dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value --resource-type deployment --process-type web ``` -To unset an annotation, pass an empty value: - -```shell -dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment -dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment --process-type web -``` - The following resource types are supported: - `certificate` @@ -309,6 +304,52 @@ The following resource types are supported: - `traefik_ingressroute` - `traefik_middleware` +#### Removing an annotation + +To unset an annotation, pass an empty value: + +```shell +dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment +dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment --process-type web +``` + +#### Setting Labels + +Dokku injects certain labels into each created resource by default, but it may be necessary to inject others for tighter integration with third-party tools. The `scheduler-k3s:labels:set` command can be used to perform this task. The command takes an app name and a required `--resource-type` flag. + +```shell +dokku scheduler-k3s:labels:set node-js-app label.key label.value --resource-type deployment +``` + +If not specified, the label will be applied to all processes within an app, though it may be further scoped to a specific process type via the `--process-type` flag. + +```shell +dokku scheduler-k3s:labels:set node-js-app label.key label.value --resource-type deployment --process-type web +``` + +The following resource types are supported: + +- `certificate` +- `cronjob` +- `deployment` +- `ingress` +- `job` +- `pod` +- `secret` +- `service` +- `serviceaccount` +- `traefik_ingressroute` +- `traefik_middleware` + +#### Removing a label + +To unset an label, pass an empty value: + +```shell +dokku scheduler-k3s:annotations:set node-js-app label.key --resource-type deployment +dokku scheduler-k3s:labels:set node-js-app label.key --resource-type deployment --process-type web +``` + ### Using kubectl remotely > [!WARNING] diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index 0e3e8d2ca..c150d6aa1 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,4 +1,4 @@ -SUBCOMMANDS = subcommands/annotations:set subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall +SUBCOMMANDS = subcommands/annotations:set subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 0c32b281f..c0387b8b0 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -675,6 +675,103 @@ func getIngressAnnotations(appName string, processType string) (map[string]strin return annotations, nil } +// getLabels retrieves labels for a given app and process type +func getLabels(appName string, processType string) (ProcessLabels, error) { + labels := ProcessLabels{} + certificateLabels, err := getAnnotation(appName, processType, "certificate") + if err != nil { + return labels, err + } + labels.CertificateLabels = certificateLabels + + cronJobLabels, err := getAnnotation(appName, processType, "cronjob") + if err != nil { + return labels, err + } + labels.CronJobLabels = cronJobLabels + + deploymentLabels, err := getAnnotation(appName, processType, "deployment") + if err != nil { + return labels, err + } + labels.DeploymentLabels = deploymentLabels + + ingressLabels, err := getIngressAnnotations(appName, processType) + if err != nil { + return labels, err + } + labels.IngressLabels = ingressLabels + + jobLabels, err := getAnnotation(appName, processType, "job") + if err != nil { + return labels, err + } + labels.JobLabels = jobLabels + + podLabels, err := getAnnotation(appName, processType, "pod") + if err != nil { + return labels, err + } + labels.PodLabels = podLabels + + secretLabels, err := getAnnotation(appName, processType, "secret") + if err != nil { + return labels, err + } + labels.SecretLabels = secretLabels + + serviceLabels, err := getAnnotation(appName, processType, "service") + if err != nil { + return labels, err + } + labels.ServiceLabels = serviceLabels + + serviceAccountLabels, err := getAnnotation(appName, processType, "serviceaccount") + if err != nil { + return labels, err + } + labels.ServiceAccountLabels = serviceAccountLabels + + traefikIngressRouteLabels, err := getAnnotation(appName, processType, "traefik_ingressroute") + if err != nil { + return labels, err + } + labels.TraefikIngressRouteLabels = traefikIngressRouteLabels + + traefikMiddlewareLabels, err := getAnnotation(appName, processType, "traefik_middleware") + if err != nil { + return labels, err + } + labels.TraefikMiddlewareLabels = traefikMiddlewareLabels + + return labels, nil +} + +// getGlobalLabel retrieves global labels for a given app +func getGlobalLabel(appName string) (ProcessLabels, error) { + return getLabels(appName, GlobalProcessType) +} + +// getLabel retrieves an label for a given app, process type, and resource type +func getLabel(appName string, processType string, resourceType string) (map[string]string, error) { + labels := map[string]string{} + labelsList, err := common.PropertyListGet("scheduler-k3s", appName, fmt.Sprintf("labels.%s.%s", processType, resourceType)) + if err != nil { + return labels, err + } + + for _, label := range labelsList { + parts := strings.SplitN(label, ": ", 2) + if len(parts) != 2 { + return labels, fmt.Errorf("Invalid label format: %s", label) + } + + labels[parts[0]] = parts[1] + } + + return labels, nil +} + func getLetsencryptServer(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "letsencrypt-server", "") } diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index afef80116..4685d0834 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -60,6 +60,22 @@ func main() { ingressClass := args.String("ingress-class", "traefik", "ingress-class: ingress-class to use for all outbound traffic") args.Parse(os.Args[2:]) err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling) + case "labels:set": + args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: set a global property") + processType := args.String("process-type", "", "--process-type: scope to process-type") + resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + property := args.Arg(1) + value := args.Arg(2) + if *global { + appName = "--global" + property = args.Arg(0) + value = args.Arg(1) + } + + err = scheduler_k3s.CommandLabelsSet(appName, *processType, *resourceType, property, value) case "report": args := flag.NewFlagSet("scheduler-k3s:report", flag.ExitOnError) format := args.String("format", "stdout", "format: [ stdout | json ]") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index a01b53265..681e426ae 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -713,6 +713,47 @@ func CommandClusterRemove(nodeName string) error { return nil } +// CommandLabelsSet set or clear a scheduler-k3s label for an app +func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error { + if resourceType == "" { + return fmt.Errorf("Missing resource-type") + } + + if processType == "" { + processType = GlobalProcessType + } + + property := fmt.Sprintf("labels.%s.%s", processType, resourceType) + labelsList, err := common.PropertyListGet("scheduler-k3s", appName, property) + if err != nil { + return fmt.Errorf("Unable to get property list: %w", err) + } + + labels := []string{} + for _, annotation := range labelsList { + parts := strings.SplitN(annotation, ": ", 2) + if len(parts) != 2 { + return fmt.Errorf("Invalid annotation: %s", annotation) + } + if key == parts[0] { + continue + } + + labels = append(labels, annotation) + } + + if value != "" { + labels = append(labels, fmt.Sprintf("%s: %s", key, value)) + } + + sort.Strings(labels) + if err := common.PropertyListWrite("scheduler-k3s", appName, property, labels); err != nil { + return fmt.Errorf("Unable to write property list: %w", err) + } + + return nil +} + // CommandReport displays a scheduler-k3s report for one or more apps func CommandReport(appName string, format string, infoFlag string) error { if len(appName) == 0 { diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 5003a3965..8a2f9b35f 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -60,6 +60,7 @@ type ProcessValues struct { Args []string `yaml:"args,omitempty"` Cron ProcessCron `yaml:"cron,omitempty"` Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"` + Labels ProcessLabels `yaml:"labels,omitempty"` ProcessType ProcessType `yaml:"process_type"` Replicas int32 `yaml:"replicas"` Resources ProcessResourcesMap `yaml:"resources,omitempty"` @@ -129,6 +130,20 @@ const ( URISchemeHTTPS URIScheme = "HTTPS" ) +type ProcessLabels struct { + CertificateLabels map[string]string `yaml:"certificate,omitempty"` + CronJobLabels map[string]string `yaml:"cronjob,omitempty"` + DeploymentLabels map[string]string `yaml:"deployment,omitempty"` + IngressLabels map[string]string `yaml:"ingress,omitempty"` + JobLabels map[string]string `yaml:"job,omitempty"` + PodLabels map[string]string `yaml:"pod,omitempty"` + SecretLabels map[string]string `yaml:"secret,omitempty"` + ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` + ServiceLabels map[string]string `yaml:"service,omitempty"` + TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` + TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` +} + type ProcessWeb struct { Domains []string `yaml:"domains,omitempty"` PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"` diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 46518c1d6..abf5ace4c 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -316,10 +316,16 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting process annotations: %w", err) } + labels, err := getLabels(appName, processType) + if err != nil { + return fmt.Errorf("Error getting process labels: %w", err) + } + processValues := ProcessValues{ Annotations: annotations, Args: args, Healthchecks: processHealthchecks, + Labels: labels, ProcessType: ProcessType_Worker, Replicas: int32(processCount), Resources: processResources, @@ -458,6 +464,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting process annotations: %w", err) } + labels, err := getLabels(appName, cronEntry.ID) + if err != nil { + return fmt.Errorf("Error getting process labels: %w", err) + } + processValues := ProcessValues{ Args: words, Annotations: annotations, @@ -466,6 +477,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Schedule: cronEntry.Schedule, Suffix: suffix, }, + Labels: labels, ProcessType: ProcessType_Cron, Replicas: 1, Resources: processResources,