diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 63e33623d..6054812b8 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -4,6 +4,7 @@ > New as of 0.33.0 ``` +scheduler-k3s:annotations:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear an annotation for a given app/process-type/resource-type combination scheduler-k3s:cluster-add [ssh://user@host:port] # Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list # Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id] # Removes client node to a Dokku-managed cluster @@ -262,6 +263,44 @@ The default value may be set by passing an empty value for the option. dokku scheduler-k3s:set --global letsencrypt-server staging ``` +### Customizing Resource Annotations + +Dokku injects certain resources into each created resource by default, but it may be necessary to inject others for tighter integration with third-party tools. The `scheduler-k3s:annotations:set` command can be used to perform this task. The command takes an app name and a required `--resource-type` flag. + +```shell +dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value --resource-type deployment +``` + +If not specified, the annotation will be applied to all processes within an app, though it may be further scoped to a specific process type via the `--process-type` flag. + +> [!NOTE] +> The cron ID is used as the process type if your app deploys any cron tasks + +```shell +dokku scheduler-k3s:annotations:set node-js-app annotation.key annotation.value --resource-type deployment --process-type web +``` + +To unset an annotation, pass an empty value: + +```shell +dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment +dokku scheduler-k3s:annotations:set node-js-app annotation.key --resource-type deployment --process-type web +``` + +The following resource types are supported: + +- `certificate` +- `cronjob` +- `deployment` +- `ingress` +- `job` +- `pod` +- `secret` +- `service` +- `serviceaccount` +- `traefik_ingressroute` +- `traefik_middleware` + ### Using kubectl remotely > [!WARNING] diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 5499615dd..c783d4502 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -363,6 +363,103 @@ func extractStartCommand(input StartCommandInput) string { return command } +// getAnnotations retrieves annotations for a given app and process type +func getAnnotations(appName string, processType string) (ProcessAnnotations, error) { + annotations := ProcessAnnotations{} + certificateAnnotations, err := getAnnotation(appName, processType, "certificate") + if err != nil { + return annotations, err + } + annotations.CertificateAnnotations = certificateAnnotations + + cronJobAnnotations, err := getAnnotation(appName, processType, "cronjob") + if err != nil { + return annotations, err + } + annotations.CronJobAnnotations = cronJobAnnotations + + deploymentAnnotations, err := getAnnotation(appName, processType, "deployment") + if err != nil { + return annotations, err + } + annotations.DeploymentAnnotations = deploymentAnnotations + + ingressAnnotations, err := getAnnotation(appName, processType, "ingress") + if err != nil { + return annotations, err + } + annotations.IngressAnnotations = ingressAnnotations + + jobAnnotations, err := getAnnotation(appName, processType, "job") + if err != nil { + return annotations, err + } + annotations.JobAnnotations = jobAnnotations + + podAnnotations, err := getAnnotation(appName, processType, "pod") + if err != nil { + return annotations, err + } + annotations.PodAnnotations = podAnnotations + + secretAnnotations, err := getAnnotation(appName, processType, "secret") + if err != nil { + return annotations, err + } + annotations.SecretAnnotations = secretAnnotations + + serviceAnnotations, err := getAnnotation(appName, processType, "service") + if err != nil { + return annotations, err + } + annotations.ServiceAnnotations = serviceAnnotations + + serviceAccountAnnotations, err := getAnnotation(appName, processType, "serviceaccount") + if err != nil { + return annotations, err + } + annotations.ServiceAccountAnnotations = serviceAccountAnnotations + + traefikIngressRouteAnnotations, err := getAnnotation(appName, processType, "traefik_ingressroute") + if err != nil { + return annotations, err + } + annotations.TraefikIngressRouteAnnotations = traefikIngressRouteAnnotations + + traefikMiddlewareAnnotations, err := getAnnotation(appName, processType, "traefik_middleware") + if err != nil { + return annotations, err + } + annotations.TraefikMiddlewareAnnotations = traefikMiddlewareAnnotations + + return annotations, nil +} + +// getGlobalAnnotations retrieves global annotations for a given app +func getGlobalAnnotations(appName string) (ProcessAnnotations, error) { + return getAnnotations(appName, GlobalProcessType) +} + +// getAnnotation retrieves an annotation for a given app, process type, and resource type +func getAnnotation(appName string, processType string, resourceType string) (map[string]string, error) { + annotations := map[string]string{} + annotationsList, err := common.PropertyListGet("scheduler-k3s", appName, fmt.Sprintf("%s.%s", processType, resourceType)) + if err != nil { + return annotations, err + } + + for _, annotation := range annotationsList { + parts := strings.SplitN(annotation, ": ", 2) + if len(parts) != 2 { + return annotations, fmt.Errorf("Invalid annotation format: %s", annotation) + } + + annotations[parts[0]] = parts[1] + } + + return annotations, nil +} + func getDeployTimeout(appName string) string { return common.PropertyGetDefault("scheduler-k3s", appName, "deploy-timeout", "") } diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index 8c056581d..99483f883 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -117,6 +117,9 @@ func (k KubernetesClient) ApplyKubernetesManifest(ctx context.Context, input App "-f", input.Manifest, }, + Env: map[string]string{ + "KUBECONFIG": KubeConfigPath, + }, StreamStdio: true, }) if err != nil { diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 297f45b5b..8b57e389a 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -42,6 +42,8 @@ const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml" +const GlobalProcessType = "--global" + var ( runtimeScheme = runtime.NewScheme() codecs = serializer.NewCodecFactory(runtimeScheme) diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index 64594c7e8..d8071dfb7 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -1,13 +1,15 @@ package main import ( - "flag" "fmt" "os" "strconv" "strings" "github.com/dokku/dokku/plugins/common" + scheduler_k3s "github.com/dokku/dokku/plugins/scheduler-k3s" + + flag "github.com/spf13/pflag" ) const ( @@ -18,6 +20,7 @@ Manage scheduler-k3s settings for an app Additional commands:` helpContent = ` + scheduler-k3s:annotations:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear an annotation for a given app/process-type/resource-type combination scheduler-k3s:cluster-add [--insecure-allow-unknown-hosts] [--server-ip SERVER_IP] [--taint-scheduling] , Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list [--format json|stdout], Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id], Removes client node to a Dokku-managed cluster @@ -30,10 +33,11 @@ Additional commands:` ) func main() { - flag.Usage = usage - flag.Parse() + cmd := "" + if len(os.Args) > 1 { + cmd = os.Args[1] + } - cmd := flag.Arg(0) switch cmd { case "scheduler-k3s", "scheduler-k3s:help": usage() @@ -47,6 +51,28 @@ func main() { } else { fmt.Print("\n scheduler-k3s, Manage scheduler-k3s settings for an app\n") } + case "scheduler-k3s:annotations:set": + args := flag.NewFlagSet("scheduler-k3s:annotations:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: set a global property") + processType := args.String("process-type", "", "--process-type: scope to process-type") + resourceType := args.String("resource-type", "", "--resource-type: scope to resource-type") + err := args.Parse(os.Args[2:]) + if err != nil { + common.LogFailWithError(err) + } + + appName := args.Arg(0) + property := args.Arg(1) + value := args.Arg(2) + if *global { + appName = "--global" + property = args.Arg(0) + value = args.Arg(1) + } + + if err := scheduler_k3s.CommandAnnotationsSet(appName, *processType, *resourceType, property, value); err != nil { + common.LogFailWithError(err) + } default: dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT")) if err != nil { diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 003848036..ac7867eaa 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "os/signal" + "sort" "strings" "syscall" @@ -16,6 +17,47 @@ import ( "github.com/ryanuber/columnize" ) +// CommandAnnotationsSet set or clear a scheduler-k3s annotation for an app +func CommandAnnotationsSet(appName string, processType string, resourceType string, key string, value string) error { + if resourceType == "" { + return fmt.Errorf("Missing resource-type") + } + + if processType == "" { + processType = GlobalProcessType + } + + property := fmt.Sprintf("%s.%s", processType, resourceType) + annotationsList, err := common.PropertyListGet("scheduler-k3s", appName, property) + if err != nil { + return fmt.Errorf("Unable to get property list: %w", err) + } + + annotations := []string{} + for _, annotation := range annotationsList { + parts := strings.SplitN(annotation, ": ", 2) + if len(parts) != 2 { + return fmt.Errorf("Invalid annotation: %s", annotation) + } + if key == parts[0] { + continue + } + + annotations = append(annotations, annotation) + } + + if value != "" { + annotations = append(annotations, fmt.Sprintf("%s: %s", key, value)) + } + + sort.Strings(annotations) + if err := common.PropertyListWrite("scheduler-k3s", appName, property, annotations); err != nil { + return fmt.Errorf("Unable to write property list: %w", err) + } + + return nil +} + // CommandInitialize initializes a k3s cluster on the local server func CommandInitialize(serverIP string, taintScheduling bool) error { if err := isK3sInstalled(); err == nil { diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 1960a3ad7..b9c76c96a 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -33,12 +33,13 @@ type AppValues struct { } type GlobalValues struct { - AppName string `yaml:"app_name"` - DeploymentID string `yaml:"deploment_id"` - Image GlobalImage `yaml:"image"` - Namespace string `yaml:"namespace"` - Network GlobalNetwork `yaml:"network"` - Secrets map[string]string `yaml:"secrets,omitempty"` + Annotations ProcessAnnotations `yaml:"annotations,omitempty"` + AppName string `yaml:"app_name"` + DeploymentID string `yaml:"deploment_id"` + Image GlobalImage `yaml:"image"` + Namespace string `yaml:"namespace"` + Network GlobalNetwork `yaml:"network"` + Secrets map[string]string `yaml:"secrets,omitempty"` } type GlobalImage struct { @@ -54,6 +55,7 @@ type GlobalNetwork struct { } type ProcessValues struct { + Annotations ProcessAnnotations `yaml:"annotations,omitempty"` Args []string `yaml:"args,omitempty"` Cron ProcessCron `yaml:"cron,omitempty"` Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"` @@ -63,6 +65,20 @@ type ProcessValues struct { Web ProcessWeb `yaml:"web,omitempty"` } +type ProcessAnnotations struct { + CertificateAnnotations map[string]string `yaml:"certificate,omitempty"` + CronJobAnnotations map[string]string `yaml:"cronjob,omitempty"` + DeploymentAnnotations map[string]string `yaml:"deployment,omitempty"` + IngressAnnotations map[string]string `yaml:"ingress,omitempty"` + JobAnnotations map[string]string `yaml:"job,omitempty"` + PodAnnotations map[string]string `yaml:"pod,omitempty"` + SecretAnnotations map[string]string `yaml:"secret,omitempty"` + ServiceAnnotations map[string]string `yaml:"service,omitempty"` + ServiceAccountAnnotations map[string]string `yaml:"serviceaccount,omitempty"` + TraefikIngressRouteAnnotations map[string]string `yaml:"traefik_ingressroute,omitempty"` + TraefikMiddlewareAnnotations map[string]string `yaml:"traefik_middleware,omitempty"` +} + type ProcessHealthchecks struct { Liveness ProcessHealthcheck `yaml:"liveness,omitempty"` Readiness ProcessHealthcheck `yaml:"readiness,omitempty"` @@ -229,7 +245,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { n := 5 b := make([]byte, n) if _, err := rand.Read(b); err != nil { - panic(err) + return batchv1.Job{}, fmt.Errorf("Error generating random suffix: %w", err) } suffix = strings.ToLower(fmt.Sprintf("%X", b)) } @@ -238,6 +254,28 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { podAnnotations := annotations podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType) + globalAnnotations, err := getGlobalAnnotations(input.AppName) + if err != nil { + return batchv1.Job{}, fmt.Errorf("Error getting global annotations: %w", err) + } + for key, value := range globalAnnotations.JobAnnotations { + annotations[key] = value + } + for key, value := range globalAnnotations.PodAnnotations { + podAnnotations[key] = value + } + + processAnnotations, err := getAnnotations(input.AppName, input.ProcessType) + if err != nil { + return batchv1.Job{}, fmt.Errorf("Error getting process annotations: %w", err) + } + for key, value := range processAnnotations.JobAnnotations { + annotations[key] = value + } + for key, value := range processAnnotations.PodAnnotations { + podAnnotations[key] = value + } + job := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix), diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml index 796cecd71..0ee993927 100644 --- a/plugins/scheduler-k3s/templates/chart/certificate.yaml +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -7,6 +7,16 @@ kind: Certificate metadata: annotations: dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.certificate }} + {{- range $k, $v := $.Values.global.annotations.certificate }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.certificate }} + {{- range $k, $v := $config.annotations.certificate }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index 2dca28242..73cceec23 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -11,6 +11,16 @@ metadata: dokku.com/job-suffix: {{ $config.cron.suffix }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + {{- if and $.Values.global.annotations $.Values.global.annotations.cronjob }} + {{- range $k, $v := $.Values.global.annotations.cronjob }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.cronjob }} + {{- range $k, $v := $config.annotations.cronjob }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} app.kubernetes.io/name: cron @@ -30,6 +40,16 @@ spec: dokku.com/job-suffix: {{ $config.cron.suffix }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + {{- if and $.Values.global.annotations $.Values.global.annotations.pod }} + {{- range $k, $v := $.Values.global.annotations.pod }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.pod }} + {{- range $k, $v := $config.annotations.pod }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} app.kubernetes.io/name: cron diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index 0c633290a..d064996fd 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -9,6 +9,16 @@ metadata: dokku.com/builder-type: {{ $.Values.global.image.type }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }} + {{- if and $.Values.global.annotations $.Values.global.annotations.deployment }} + {{- range $k, $v := $.Values.global.annotations.deployment }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.deployment }} + {{- range $k, $v := $config.annotations.deployment }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} @@ -31,6 +41,16 @@ spec: dokku.com/builder-type: {{ $.Values.global.image.type }} dokku.com/managed: "true" kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }} + {{- if and $.Values.global.annotations $.Values.global.annotations.pod }} + {{- range $k, $v := $.Values.global.annotations.pod }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.pod }} + {{- range $k, $v := $config.annotations.pod }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} diff --git a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml index e65ba3b1f..fef18ded1 100644 --- a/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml +++ b/plugins/scheduler-k3s/templates/chart/https-redirect-middleware.yaml @@ -7,6 +7,16 @@ kind: Middleware metadata: annotations: dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.traefik_middleware }} + {{- range $k, $v := $.Values.global.annotations.traefik_middleware }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.traefik_middleware }} + {{- range $k, $v := $config.annotations.traefik_middleware }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}-redirect-to-https app.kubernetes.io/name: {{ $processName }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index 8c1db08da..ca9e3f87e 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -8,6 +8,16 @@ kind: IngressRoute metadata: annotations: dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.traefik_ingressroute }} + {{- range $k, $v := $.Values.global.annotations.traefik_ingressroute }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.traefik_ingressroute }} + {{- range $k, $v := $config.annotations.traefik_ingressroute }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} diff --git a/plugins/scheduler-k3s/templates/chart/secret.yaml b/plugins/scheduler-k3s/templates/chart/secret.yaml index d3f1e02ce..af4f8131c 100644 --- a/plugins/scheduler-k3s/templates/chart/secret.yaml +++ b/plugins/scheduler-k3s/templates/chart/secret.yaml @@ -4,6 +4,11 @@ metadata: annotations: app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.secret }} + {{- range $k, $v := $.Values.global.annotations.secret }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} app.kubernetes.io/name: env-{{ $.Values.global.app_name }} diff --git a/plugins/scheduler-k3s/templates/chart/service-account.yaml b/plugins/scheduler-k3s/templates/chart/service-account.yaml index 12cdc0717..eaf142e84 100644 --- a/plugins/scheduler-k3s/templates/chart/service-account.yaml +++ b/plugins/scheduler-k3s/templates/chart/service-account.yaml @@ -1,6 +1,13 @@ apiVersion: v1 kind: ServiceAccount metadata: + annotations: + dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.serviceaccount }} + {{- range $k, $v := $.Values.global.annotations.serviceaccount }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: service-account app.kubernetes.io/name: service-account diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml index 3d5619e8c..d6b812762 100644 --- a/plugins/scheduler-k3s/templates/chart/service.yaml +++ b/plugins/scheduler-k3s/templates/chart/service.yaml @@ -6,6 +6,16 @@ kind: Service metadata: annotations: dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.service }} + {{- range $k, $v := $.Values.global.annotations.service }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if and $config.annotations $config.annotations.service }} + {{- range $k, $v := $config.annotations.service }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index b4b4f0b7e..723cf81b7 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -333,8 +333,14 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error writing chart: %w", err) } + globalAnnotations, err := getGlobalAnnotations(appName) + if err != nil { + return fmt.Errorf("Error getting global annotations: %w", err) + } + values := &AppValues{ Global: GlobalValues{ + Annotations: globalAnnotations, AppName: appName, DeploymentID: fmt.Sprint(deploymentId), Image: GlobalImage{ @@ -381,7 +387,13 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting process resources: %w", err) } + annotations, err := getAnnotations(appName, processType) + if err != nil { + return fmt.Errorf("Error getting process annotations: %w", err) + } + processValues := ProcessValues{ + Annotations: annotations, Args: args, Healthchecks: processHealthchecks, ProcessType: ProcessType_Worker, @@ -494,8 +506,14 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting process resources: %w", err) } + annotations, err := getAnnotations(appName, cronEntry.ID) + if err != nil { + return fmt.Errorf("Error getting process annotations: %w", err) + } + processValues := ProcessValues{ - Args: words, + Args: words, + Annotations: annotations, Cron: ProcessCron{ ID: cronEntry.ID, Schedule: cronEntry.Schedule, @@ -1237,9 +1255,9 @@ func TriggerSchedulerStop(scheduler string, appName string) error { } for _, deployment := range deployments { - processType, ok := deployment.Annotations["app.kubernetes.io/name"] + processType, ok := deployment.Labels["app.kubernetes.io/name"] if !ok { - return fmt.Errorf("Deployment %s does not have a process type annotation", deployment.Name) + return fmt.Errorf("Deployment %s does not have a process type label", deployment.Name) } common.LogVerboseQuiet(fmt.Sprintf("Stopping %s process", processType)) err := clientset.ScaleDeployment(ctx, ScaleDeploymentInput{ diff --git a/tests/unit/scheduler-k3s.bats b/tests/unit/scheduler-k3s.bats new file mode 100644 index 000000000..ca503a258 --- /dev/null +++ b/tests/unit/scheduler-k3s.bats @@ -0,0 +1,181 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown_() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +install_k3s() { + run /bin/bash -c "dokku proxy:set --global k3s" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:set --global server hub.docker.com" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:set --global image-repo-template '$DOCKERHUB_USERNAME/{{ .AppName }}'" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:set --global push-on-release true" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler:set --global selected k3s" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku registry:login docker.io $DOCKERHUB_USERNAME $DOCKERHUB_TOKEN" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Login Succeeded" + + run /bin/bash -c "dokku scheduler-k3s:initialize" + echo "output: $output" + echo "status: $status" + assert_success +} + +uninstall_k3s() { + run /bin/bash -c "dokku scheduler-k3s:uninstall" + echo "output: $output" + echo "status: $status" + assert_success +} + +@test "(scheduler-k3s) install" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + install_k3s +} + +@test "(scheduler-k3s) deploy" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 20" + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" +} + +@test "(scheduler-k3s) deploy annotations" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type secret test.dokku.com/resource-type secret" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type pod test.dokku.com/resource-type pod" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --resource-type deployment test.dokku.com/resource-type deployment" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:annotations:set $TEST_APP --process-type web --resource-type deployment test.dokku.com/resource-type deployment-web" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:sync --build $TEST_APP https://github.com/dokku/smoke-test-app.git" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 20" + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.metadata.annotations.\"test.dokku.com/resource-type\"'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "deployment-web" + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.metadata.annotations.\"test.dokku.com/resource-type\"'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "pod" + + run /bin/bash -c "kubectl get cronjob -o json | jq -r '.items[0].metadata.annotations.\"test.dokku.com/resource-type\"'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "null" + + run /bin/bash -c "kubectl get cronjob -o json | jq -r '.items[0].spec.jobTemplate.metadata.annotations.\"test.dokku.com/resource-type\"'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "pod" + + run /bin/bash -c "kubectl get secret -o json | jq -r '.items[0].metadata.annotations.\"test.dokku.com/resource-type\"'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "secret" +}