From b98863b6a7bb8c80251fea50cc431b126685321f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 03:29:26 -0500 Subject: [PATCH] feat: add support for the keda-http-add-on Externally, it is treated the same as any other scaling trigger, but requires that the chart be installed in order for it to be supported. --- docs/deployment/schedulers/k3s.md | 50 +++++++++++++++++ plugins/scheduler-k3s/functions.go | 49 ++++++++++++++++- plugins/scheduler-k3s/template.go | 40 ++++++++++---- .../templates/chart/ingress.yaml | 14 +++++ .../chart/keda-http-scaled-object.yaml | 55 +++++++++++++++++++ .../chart/keda-interceptor-proxy-service.yaml | 26 +++++++++ plugins/scheduler-k3s/triggers.go | 9 ++- 7 files changed, 227 insertions(+), 16 deletions(-) create mode 100644 plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml create mode 100644 plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index c1c4e705c..873406b5a 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -417,6 +417,56 @@ Each value in the `metadata` stanza can use the following interpolated strings: - `DOKKU_PROCESS_TYPE`: The name of the process being scaled - `DOKKU_APP_NAME`: The name of the app being scaled +##### HTTP Autoscaling + +In addition to the built-in scalers that Keda provides, Dokku also supports Keda's HTTP Add On. This requires that the addon be properly installed and configured. For existing k3s clusters, this can be performed by the `scheduler-k3s:ensure-charts` command: + +```shell +dokku scheduler-k3s:ensure-charts +``` + +> [!NOTE] +> Users who wish to use this functionality on a cluster not managed by Dokku will need to manually install the `keda-http-add-on` into the `keda` namespace. Please consult the `keda-http-add-on` [install documentation](https://kedacore.github.io/http-add-on/install.html) for further details. + +> [!WARNING] +> If the `keda-http-add-on` chart is not installed, then this trigger will be ignored. + +Once the chart is configured, an `http` trigger can be specified like so: + + +```json +{ + "formation": { + "web": { + "autoscaling": { + "min_quantity": 1, + "max_quantity": 10, + "triggers": [ + { + "type": "http", + "metadata": { + "scaledown_period_seconds": "150", + "request_rate_target_value": "50" + } + } + ] + } + } + } +} +``` + +The following metadata properties are supported with the http autoscaler: + +- `scale_by`: (default: `request_rate`) whether to scale by `concurrency` or `request_rate`. +- `scaledown_period_seconds`: (default: `300`) period to wait after the last reported active before scaling the resource back to 0. +- `request_rate_granularity_seconds`: (default: `1`) granualarity of the aggregated requests for the request rate calculation. +- `request_rate_target_value`: (default: `100`) target value for the request rate. +- `request_rate_window_seconds`: (default: `60`) aggregation window for the request rate calculation. +- `concurrency_target_value`: (default: `100`) target value for the request concurrency. + +Note that due to Keda limitations, scaling is done by _either_ `concurrency` or `request_rate`. + #### Workload Autoscaling Authentication Most Keda triggers require some form of authentication to query for data. In the Kubernetes API, they are represented by `TriggerAuthentication` and `ClusterTriggerAuthentication` resources. Dokku can manage these via the `scheduler-k3s:autoscaling-auth` commands, and includes generated resources with each helm release generated by a deploy. diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index e3b0f42ca..4f84edca3 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -584,6 +584,13 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { "DEPLOYMENT_NAME": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), } + validHttpScaleMethods := map[string]bool{ + "request_rate": true, + "concurrency": true, + } + + hasHttpTrigger := false + httpTrigger := ProcessAutoscalingTrigger{} triggers := []ProcessAutoscalingTrigger{} for idx, trigger := range config.Triggers { if trigger.Type == "" { @@ -625,15 +632,55 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { } } + if autoscalingTrigger.Type == "http" { + if hasHttpTrigger { + return ProcessAutoscaling{}, errors.New("Only one http trigger is allowed") + } + + hasHttpTrigger = true + httpTrigger = autoscalingTrigger + + if _, ok := metadata["scale_by"]; !ok { + httpTrigger.Metadata["scale_by"] = "request_rate" + } + + if !validHttpScaleMethods[metadata["scale_by"]] { + return ProcessAutoscaling{}, fmt.Errorf("Invalid http scale method: %s", metadata["scale_by"]) + } + + if _, ok := metadata["scaledown_period_seconds"]; !ok { + httpTrigger.Metadata["scaledown_period_seconds"] = "300" + } + + if _, ok := metadata["request_rate_granularity_seconds"]; !ok { + httpTrigger.Metadata["request_rate_granularity_seconds"] = "1" + } + + if _, ok := metadata["request_rate_target_value"]; !ok { + httpTrigger.Metadata["request_rate_target_value"] = "100" + } + + if _, ok := metadata["request_rate_window_seconds"]; !ok { + httpTrigger.Metadata["request_rate_window_seconds"] = "60" + } + + if _, ok := metadata["concurrency_target_value"]; !ok { + httpTrigger.Metadata["concurrency_target_value"] = "100" + } + + continue + } + triggers = append(triggers, autoscalingTrigger) } autoscaling := ProcessAutoscaling{ CooldownPeriodSeconds: ptr.Deref(config.CooldownPeriodSeconds, 300), - Enabled: len(triggers) > 0, + Enabled: len(triggers) > 0 || hasHttpTrigger, MaxReplicas: ptr.Deref(config.MaxQuantity, 0), MinReplicas: ptr.Deref(config.MinQuantity, 0), PollingIntervalSeconds: ptr.Deref(config.PollingIntervalSeconds, 30), + HttpTrigger: httpTrigger, Triggers: triggers, Type: "keda", } diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index abc628ab3..17b5acaa5 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -61,8 +61,14 @@ type GlobalImage struct { } type GlobalNetwork struct { + // IngressClass is the default ingress class to use IngressClass string `yaml:"ingress_class"` - PrimaryPort int32 `yaml:"primary_port"` + + // PrimaryPort is the primary port to use + PrimaryPort int32 `yaml:"primary_port"` + + // PrimaryServicePort is the primary service port to use + PrimaryServicePort int32 `yaml:"primary_service_port"` } // GlobalKedaValues contains the global keda configuration @@ -116,6 +122,8 @@ type ProcessAnnotations struct { IngressAnnotations map[string]string `yaml:"ingress,omitempty"` JobAnnotations map[string]string `yaml:"job,omitempty"` KedaScalingObjectAnnotations map[string]string `yaml:"keda_scaled_object,omitempty"` + KedaHTTPScaledObjectAnnotations map[string]string `yaml:"keda_http_scaled_object,omitempty"` + KedaInterceptorProxyAnnotations map[string]string `yaml:"keda_interceptor_proxy,omitempty"` KedaSecretAnnotations map[string]string `yaml:"keda_secret,omitempty"` KedaTriggerAuthenticationAnnotations map[string]string `yaml:"keda_trigger_authentication,omitempty"` PodAnnotations map[string]string `yaml:"pod,omitempty"` @@ -143,6 +151,9 @@ type ProcessAutoscaling struct { // PollingIntervalSeconds is the number of seconds between polling for new metrics PollingIntervalSeconds int `yaml:"polling_interval_seconds,omitempty"` + // HttpTrigger is the http trigger config to use for autoscaling + HttpTrigger ProcessAutoscalingTrigger `yaml:"http_trigger,omitempty"` + // Triggers is a list of triggers to use for autoscaling Triggers []ProcessAutoscalingTrigger `yaml:"triggers,omitempty"` @@ -224,17 +235,22 @@ const ( ) type ProcessLabels struct { - CertificateLabels map[string]string `yaml:"certificate,omitempty"` - CronJobLabels map[string]string `yaml:"cronjob,omitempty"` - DeploymentLabels map[string]string `yaml:"deployment,omitempty"` - IngressLabels map[string]string `yaml:"ingress,omitempty"` - JobLabels map[string]string `yaml:"job,omitempty"` - PodLabels map[string]string `yaml:"pod,omitempty"` - SecretLabels map[string]string `yaml:"secret,omitempty"` - ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` - ServiceLabels map[string]string `yaml:"service,omitempty"` - TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` - TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` + CertificateLabels map[string]string `yaml:"certificate,omitempty"` + CronJobLabels map[string]string `yaml:"cronjob,omitempty"` + DeploymentLabels map[string]string `yaml:"deployment,omitempty"` + IngressLabels map[string]string `yaml:"ingress,omitempty"` + JobLabels map[string]string `yaml:"job,omitempty"` + KedaScalingObjectLabels map[string]string `yaml:"keda_scaled_object,omitempty"` + KedaHTTPScaledObjectLabels map[string]string `yaml:"keda_http_scaled_object,omitempty"` + KedaInterceptorProxyLabels map[string]string `yaml:"keda_interceptor_proxy,omitempty"` + KedaSecretLabels map[string]string `yaml:"keda_secret,omitempty"` + KedaTriggerAuthenticationLabels map[string]string `yaml:"keda_trigger_authentication,omitempty"` + PodLabels map[string]string `yaml:"pod,omitempty"` + SecretLabels map[string]string `yaml:"secret,omitempty"` + ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` + ServiceLabels map[string]string `yaml:"service,omitempty"` + TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` + TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` } type ProcessWeb struct { diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index b0eaa6c57..fbe512c17 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,5 +1,11 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} +{{- $routeToKeda := false }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") $config.autoscaling.http_trigger.type }} +{{- $routeToKeda = true }} +{{- end }} +{{- end }} {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} {{- range $pdx, $domain := $config.web.domains }} --- @@ -36,11 +42,19 @@ spec: paths: {{- if $config.web.port_maps }} - backend: + {{- if $routeToKeda}} + service: + name: kip-{{ $.Values.global.app_name}}-{{ $processName }} + port: + number: 8080 + pathType: Prefix + {{- else }} service: name: {{ $.Values.global.app_name }}-{{ $processName }} port: name: {{ include "primary.port" $config.web.port_maps }} pathType: ImplementationSpecific + {{- end }} path: / {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml new file mode 100644 index 000000000..e0a12c70b --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -0,0 +1,55 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := index .Values.processes "PROCESS_NAME" }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} +{{- if $config.autoscaling.http_trigger.type }} +{{- $mappings := dict }} +{{- range $pdx, $port_map := $config.web.port_maps }} +{{- $mappings := set $mappings $port_map.name "true" }} +{{- end }} +--- +kind: HTTPScaledObject +apiVersion: http.keda.sh/v1alpha1 +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} + dokku.com/managed: "true" + httpscaledobject.keda.sh/skip-scaledobject-creation: "true" + {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }} + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }} + name: {{ $.Values.global.app_name}}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} +spec: + hosts: + {{- range $pdx, $domain := $config.web.domains }} + - {{ $domain.name | quote }} + {{- end }} + scaleTargetRef: + name: {{ $.Values.global.app_name }}-{{ $processName }} + kind: Deployment + apiVersion: apps/v1 + service: {{ $.Values.global.app_name }}-{{ $processName }} + port: {{ $.Values.global.network.primary_service_port }} + replicas: + min: {{ $config.autoscaling.min_replicas }} + max: {{ $config.autoscaling.max_replicas }} + scaledownPeriod: {{ get $config.autoscaling.http_trigger.metadata "scaledown_period_seconds" }} + scalingMetric: + {{- if eq (get $config.autoscaling.http_trigger.metadata "scale_by") "request_rate" }} + requestRate: + granularity: {{ get $config.autoscaling.http_trigger.metadata "request_rate_granularity_seconds" }}s + targetValue: {{ get $config.autoscaling.http_trigger.metadata "request_rate_target_value" }} + window: {{ get $config.autoscaling.http_trigger.metadata "request_rate_window_seconds" }}s + {{- else }} + concurrency: + targetValue: {{ get $config.autoscaling.http_trigger.metadata "concurrency_target_value" }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml new file mode 100644 index 000000000..c12d6411e --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml @@ -0,0 +1,26 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := index .Values.processes "PROCESS_NAME" }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} +--- +kind: Service +apiVersion: v1 +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} + dokku.com/managed: "true" + {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }} + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }} + name: kip-{{ $.Values.global.app_name}}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} +spec: + type: ExternalName + externalName: keda-add-ons-http-interceptor-proxy.keda.svc.cluster.local +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 1edafd3ae..6763d1ab8 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -250,8 +250,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } primaryPort := int32(5000) + primaryServicePort := int32(80) for _, portMap := range portMaps { primaryPort = portMap.ContainerPort + primaryServicePort = portMap.HostPort if primaryPort != 0 { break } @@ -350,8 +352,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Labels: globalLabels, Namespace: namespace, Network: GlobalNetwork{ - IngressClass: getGlobalIngressClass(), - PrimaryPort: primaryPort, + IngressClass: getGlobalIngressClass(), + PrimaryPort: primaryPort, + PrimaryServicePort: primaryServicePort, }, Secrets: map[string]string{}, }, @@ -504,7 +507,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e templateFiles := []string{"deployment", "keda-scaled-object"} if processType == "web" { - templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware") + templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service") } for _, templateName := range templateFiles { b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))