feat: add support for the keda-http-add-on

Externally, it is treated the same as any other scaling trigger, but requires that the chart be installed in order for it to be supported.
2024-11-13 03:29:26 -05:00
parent 1751bf30ba
commit b98863b6a7
7 changed files with 227 additions and 16 deletions
--- a/docs/deployment/schedulers/k3s.md
+++ b/docs/deployment/schedulers/k3s.md
@@ -417,6 +417,56 @@ Each value in the `metadata` stanza can use the following interpolated strings:
 - `DOKKU_PROCESS_TYPE`: The name of the process being scaled
 - `DOKKU_APP_NAME`: The name of the app being scaled

+##### HTTP Autoscaling
+
+In addition to the built-in scalers that Keda provides, Dokku also supports Keda's HTTP Add On. This requires that the addon be properly installed and configured. For existing k3s clusters, this can be performed by the `scheduler-k3s:ensure-charts` command:
+
+```shell
+dokku scheduler-k3s:ensure-charts
+```
+
+> [!NOTE]
+> Users who wish to use this functionality on a cluster not managed by Dokku will need to manually install the `keda-http-add-on` into the `keda` namespace. Please consult the `keda-http-add-on` [install documentation](https://kedacore.github.io/http-add-on/install.html) for further details.
+
+> [!WARNING]
+> If the `keda-http-add-on` chart is not installed, then this trigger will be ignored.
+
+Once the chart is configured, an `http` trigger can be specified like so:
+
+
+```json
+{
+    "formation": {
+        "web": {
+            "autoscaling": {
+                "min_quantity": 1,
+                "max_quantity": 10,
+                "triggers": [
+                    {
+                        "type": "http",
+                        "metadata": {
+                            "scaledown_period_seconds": "150",
+                            "request_rate_target_value": "50"
+                        }
+                    }
+                ]
+            }
+        }
+    }
+}
+```
+
+The following metadata properties are supported with the http autoscaler:
+
+- `scale_by`: (default: `request_rate`) whether to scale by `concurrency` or `request_rate`.
+- `scaledown_period_seconds`: (default: `300`) period to wait after the last reported active before scaling the resource back to 0.
+- `request_rate_granularity_seconds`: (default: `1`) granualarity of the aggregated requests for the request rate calculation.
+- `request_rate_target_value`: (default: `100`) target value for the request rate.
+- `request_rate_window_seconds`: (default: `60`) aggregation window for the request rate calculation.
+- `concurrency_target_value`: (default: `100`) target value for the request concurrency.
+
+Note that due to Keda limitations, scaling is done by _either_ `concurrency` or `request_rate`.
+
 #### Workload Autoscaling Authentication

 Most Keda triggers require some form of authentication to query for data. In the Kubernetes API, they are represented by `TriggerAuthentication` and `ClusterTriggerAuthentication` resources. Dokku can manage these via the `scheduler-k3s:autoscaling-auth` commands, and includes generated resources with each helm release generated by a deploy.
--- a/plugins/scheduler-k3s/functions.go
+++ b/plugins/scheduler-k3s/functions.go
@@ -584,6 +584,13 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) {
 		"DEPLOYMENT_NAME": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
 	}

+	validHttpScaleMethods := map[string]bool{
+		"request_rate": true,
+		"concurrency":  true,
+	}
+
+	hasHttpTrigger := false
+	httpTrigger := ProcessAutoscalingTrigger{}
 	triggers := []ProcessAutoscalingTrigger{}
 	for idx, trigger := range config.Triggers {
 		if trigger.Type == "" {
@@ -625,15 +632,55 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) {
 			}
 		}

+		if autoscalingTrigger.Type == "http" {
+			if hasHttpTrigger {
+				return ProcessAutoscaling{}, errors.New("Only one http trigger is allowed")
+			}
+
+			hasHttpTrigger = true
+			httpTrigger = autoscalingTrigger
+
+			if _, ok := metadata["scale_by"]; !ok {
+				httpTrigger.Metadata["scale_by"] = "request_rate"
+			}
+
+			if !validHttpScaleMethods[metadata["scale_by"]] {
+				return ProcessAutoscaling{}, fmt.Errorf("Invalid http scale method: %s", metadata["scale_by"])
+			}
+
+			if _, ok := metadata["scaledown_period_seconds"]; !ok {
+				httpTrigger.Metadata["scaledown_period_seconds"] = "300"
+			}
+
+			if _, ok := metadata["request_rate_granularity_seconds"]; !ok {
+				httpTrigger.Metadata["request_rate_granularity_seconds"] = "1"
+			}
+
+			if _, ok := metadata["request_rate_target_value"]; !ok {
+				httpTrigger.Metadata["request_rate_target_value"] = "100"
+			}
+
+			if _, ok := metadata["request_rate_window_seconds"]; !ok {
+				httpTrigger.Metadata["request_rate_window_seconds"] = "60"
+			}
+
+			if _, ok := metadata["concurrency_target_value"]; !ok {
+				httpTrigger.Metadata["concurrency_target_value"] = "100"
+			}
+
+			continue
+		}
+
 		triggers = append(triggers, autoscalingTrigger)
 	}

 	autoscaling := ProcessAutoscaling{
 		CooldownPeriodSeconds:  ptr.Deref(config.CooldownPeriodSeconds, 300),
-		Enabled:                len(triggers) > 0,
+		Enabled:                len(triggers) > 0 || hasHttpTrigger,
 		MaxReplicas:            ptr.Deref(config.MaxQuantity, 0),
 		MinReplicas:            ptr.Deref(config.MinQuantity, 0),
 		PollingIntervalSeconds: ptr.Deref(config.PollingIntervalSeconds, 30),
+		HttpTrigger:            httpTrigger,
 		Triggers:               triggers,
 		Type:                   "keda",
 	}
--- a/plugins/scheduler-k3s/template.go
+++ b/plugins/scheduler-k3s/template.go
@@ -61,8 +61,14 @@ type GlobalImage struct {
 }

 type GlobalNetwork struct {
+	// IngressClass is the default ingress class to use
 	IngressClass string `yaml:"ingress_class"`
-	PrimaryPort  int32  `yaml:"primary_port"`
+
+	// PrimaryPort is the primary port to use
+	PrimaryPort int32 `yaml:"primary_port"`
+
+	// PrimaryServicePort is the primary service port to use
+	PrimaryServicePort int32 `yaml:"primary_service_port"`
 }

 // GlobalKedaValues contains the global keda configuration
@@ -116,6 +122,8 @@ type ProcessAnnotations struct {
 	IngressAnnotations                   map[string]string `yaml:"ingress,omitempty"`
 	JobAnnotations                       map[string]string `yaml:"job,omitempty"`
 	KedaScalingObjectAnnotations         map[string]string `yaml:"keda_scaled_object,omitempty"`
+	KedaHTTPScaledObjectAnnotations      map[string]string `yaml:"keda_http_scaled_object,omitempty"`
+	KedaInterceptorProxyAnnotations      map[string]string `yaml:"keda_interceptor_proxy,omitempty"`
 	KedaSecretAnnotations                map[string]string `yaml:"keda_secret,omitempty"`
 	KedaTriggerAuthenticationAnnotations map[string]string `yaml:"keda_trigger_authentication,omitempty"`
 	PodAnnotations                       map[string]string `yaml:"pod,omitempty"`
@@ -143,6 +151,9 @@ type ProcessAutoscaling struct {
 	// PollingIntervalSeconds is the number of seconds between polling for new metrics
 	PollingIntervalSeconds int `yaml:"polling_interval_seconds,omitempty"`

+	// HttpTrigger is the http trigger config to use for autoscaling
+	HttpTrigger ProcessAutoscalingTrigger `yaml:"http_trigger,omitempty"`
+
 	// Triggers is a list of triggers to use for autoscaling
 	Triggers []ProcessAutoscalingTrigger `yaml:"triggers,omitempty"`

@@ -224,17 +235,22 @@ const (
 )

 type ProcessLabels struct {
-	CertificateLabels         map[string]string `yaml:"certificate,omitempty"`
-	CronJobLabels             map[string]string `yaml:"cronjob,omitempty"`
-	DeploymentLabels          map[string]string `yaml:"deployment,omitempty"`
-	IngressLabels             map[string]string `yaml:"ingress,omitempty"`
-	JobLabels                 map[string]string `yaml:"job,omitempty"`
-	PodLabels                 map[string]string `yaml:"pod,omitempty"`
-	SecretLabels              map[string]string `yaml:"secret,omitempty"`
-	ServiceAccountLabels      map[string]string `yaml:"serviceaccount,omitempty"`
-	ServiceLabels             map[string]string `yaml:"service,omitempty"`
-	TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"`
-	TraefikMiddlewareLabels   map[string]string `yaml:"traefik_middleware,omitempty"`
+	CertificateLabels               map[string]string `yaml:"certificate,omitempty"`
+	CronJobLabels                   map[string]string `yaml:"cronjob,omitempty"`
+	DeploymentLabels                map[string]string `yaml:"deployment,omitempty"`
+	IngressLabels                   map[string]string `yaml:"ingress,omitempty"`
+	JobLabels                       map[string]string `yaml:"job,omitempty"`
+	KedaScalingObjectLabels         map[string]string `yaml:"keda_scaled_object,omitempty"`
+	KedaHTTPScaledObjectLabels      map[string]string `yaml:"keda_http_scaled_object,omitempty"`
+	KedaInterceptorProxyLabels      map[string]string `yaml:"keda_interceptor_proxy,omitempty"`
+	KedaSecretLabels                map[string]string `yaml:"keda_secret,omitempty"`
+	KedaTriggerAuthenticationLabels map[string]string `yaml:"keda_trigger_authentication,omitempty"`
+	PodLabels                       map[string]string `yaml:"pod,omitempty"`
+	SecretLabels                    map[string]string `yaml:"secret,omitempty"`
+	ServiceAccountLabels            map[string]string `yaml:"serviceaccount,omitempty"`
+	ServiceLabels                   map[string]string `yaml:"service,omitempty"`
+	TraefikIngressRouteLabels       map[string]string `yaml:"traefik_ingressroute,omitempty"`
+	TraefikMiddlewareLabels         map[string]string `yaml:"traefik_middleware,omitempty"`
 }

 type ProcessWeb struct {
--- a/plugins/scheduler-k3s/templates/chart/ingress.yaml
+++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml
@@ -1,5 +1,11 @@
 {{- $processName := "PROCESS_NAME" }}
 {{- $config := index .Values.processes "PROCESS_NAME" }}
+{{- $routeToKeda := false }}
+{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
+{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") $config.autoscaling.http_trigger.type }}
+{{- $routeToKeda = true }}
+{{- end }}
+{{- end }}
 {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }}
 {{- range $pdx, $domain := $config.web.domains }}
 ---
@@ -36,11 +42,19 @@ spec:
        paths:
          {{- if $config.web.port_maps }}
          - backend:
+            {{- if $routeToKeda}}
+              service:
+                name: kip-{{ $.Values.global.app_name}}-{{ $processName }}
+                port:
+                  number: 8080
+            pathType: Prefix
+            {{- else }}
              service:
                name: {{ $.Values.global.app_name }}-{{ $processName }}
                port:
                  name: {{ include "primary.port" $config.web.port_maps }}
            pathType: ImplementationSpecific
+            {{- end }}
            path: /
          {{- end }}
 {{- end }}
--- a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml
+++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml
@@ -0,0 +1,55 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := index .Values.processes "PROCESS_NAME" }}
+{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
+{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}}
+{{- if $config.autoscaling.http_trigger.type }}
+{{- $mappings := dict }}
+{{- range $pdx, $port_map := $config.web.port_maps }}
+{{- $mappings := set $mappings $port_map.name "true" }}
+{{- end }}
+---
+kind: HTTPScaledObject
+apiVersion: http.keda.sh/v1alpha1
+metadata:
+  annotations:
+    app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
+    dokku.com/managed: "true"
+    httpscaledobject.keda.sh/skip-scaledobject-creation: "true"
+    {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }}
+    {{ include "print.annotations" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }}
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+    {{ include "print.labels" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }}
+    {{ include "print.labels" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }}
+  name: {{ $.Values.global.app_name}}-{{ $processName }}
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  hosts:
+    {{- range $pdx, $domain := $config.web.domains }}
+    - {{ $domain.name | quote }}
+    {{- end }}
+  scaleTargetRef:
+    name: {{ $.Values.global.app_name }}-{{ $processName }}
+    kind: Deployment
+    apiVersion: apps/v1
+    service: {{ $.Values.global.app_name }}-{{ $processName }}
+    port: {{ $.Values.global.network.primary_service_port }}
+  replicas:
+    min: {{ $config.autoscaling.min_replicas }}
+    max: {{ $config.autoscaling.max_replicas }}
+  scaledownPeriod: {{ get $config.autoscaling.http_trigger.metadata "scaledown_period_seconds" }}
+  scalingMetric:
+    {{- if eq (get $config.autoscaling.http_trigger.metadata "scale_by") "request_rate" }}
+    requestRate:
+      granularity: {{ get $config.autoscaling.http_trigger.metadata "request_rate_granularity_seconds" }}s
+      targetValue: {{ get $config.autoscaling.http_trigger.metadata "request_rate_target_value" }}
+      window: {{ get $config.autoscaling.http_trigger.metadata "request_rate_window_seconds" }}s
+    {{- else }}
+    concurrency:
+      targetValue: {{ get $config.autoscaling.http_trigger.metadata "concurrency_target_value" }}
+    {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
--- a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml
+++ b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml
@@ -0,0 +1,26 @@
+{{- $processName := "PROCESS_NAME" }}
+{{- $config := index .Values.processes "PROCESS_NAME" }}
+{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
+{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}}
+---
+kind: Service
+apiVersion: v1
+metadata:
+  annotations:
+    app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
+    dokku.com/managed: "true"
+    {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }}
+    {{ include "print.annotations" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }}
+  labels:
+    app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
+    app.kubernetes.io/name: {{ $processName }}
+    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
+    {{ include "print.labels" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }}
+    {{ include "print.labels" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }}
+  name: kip-{{ $.Values.global.app_name}}-{{ $processName }}
+  namespace: {{ $.Values.global.namespace }}
+spec:
+  type: ExternalName
+  externalName: keda-add-ons-http-interceptor-proxy.keda.svc.cluster.local
+{{- end }}
+{{- end }}
--- a/plugins/scheduler-k3s/triggers.go
+++ b/plugins/scheduler-k3s/triggers.go
@@ -250,8 +250,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	}

 	primaryPort := int32(5000)
+	primaryServicePort := int32(80)
 	for _, portMap := range portMaps {
 		primaryPort = portMap.ContainerPort
+		primaryServicePort = portMap.HostPort
 		if primaryPort != 0 {
 			break
 		}
@@ -350,8 +352,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			Labels:    globalLabels,
 			Namespace: namespace,
 			Network: GlobalNetwork{
-				IngressClass: getGlobalIngressClass(),
-				PrimaryPort:  primaryPort,
+				IngressClass:       getGlobalIngressClass(),
+				PrimaryPort:        primaryPort,
+				PrimaryServicePort: primaryServicePort,
 			},
 			Secrets: map[string]string{},
 		},
@@ -504,7 +507,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e

 		templateFiles := []string{"deployment", "keda-scaled-object"}
 		if processType == "web" {
-			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware")
+			templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
 		}
 		for _, templateName := range templateFiles {
 			b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))