diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index f1c66cee5..873406b5a 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -10,6 +10,7 @@ scheduler-k3s:autoscaling-auth:report [--format stdout|json] [--i scheduler-k3s:cluster-add [ssh://user@host:port] # Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list # Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id] # Removes client node to a Dokku-managed cluster +scheduler-k3s:ensure-charts # Ensures the k3s charts are installed scheduler-k3s:initialize # Initializes a cluster scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination scheduler-k3s:report [] [] # Displays a scheduler-k3s report for one or more apps @@ -416,6 +417,56 @@ Each value in the `metadata` stanza can use the following interpolated strings: - `DOKKU_PROCESS_TYPE`: The name of the process being scaled - `DOKKU_APP_NAME`: The name of the app being scaled +##### HTTP Autoscaling + +In addition to the built-in scalers that Keda provides, Dokku also supports Keda's HTTP Add On. This requires that the addon be properly installed and configured. For existing k3s clusters, this can be performed by the `scheduler-k3s:ensure-charts` command: + +```shell +dokku scheduler-k3s:ensure-charts +``` + +> [!NOTE] +> Users who wish to use this functionality on a cluster not managed by Dokku will need to manually install the `keda-http-add-on` into the `keda` namespace. Please consult the `keda-http-add-on` [install documentation](https://kedacore.github.io/http-add-on/install.html) for further details. + +> [!WARNING] +> If the `keda-http-add-on` chart is not installed, then this trigger will be ignored. + +Once the chart is configured, an `http` trigger can be specified like so: + + +```json +{ + "formation": { + "web": { + "autoscaling": { + "min_quantity": 1, + "max_quantity": 10, + "triggers": [ + { + "type": "http", + "metadata": { + "scaledown_period_seconds": "150", + "request_rate_target_value": "50" + } + } + ] + } + } + } +} +``` + +The following metadata properties are supported with the http autoscaler: + +- `scale_by`: (default: `request_rate`) whether to scale by `concurrency` or `request_rate`. +- `scaledown_period_seconds`: (default: `300`) period to wait after the last reported active before scaling the resource back to 0. +- `request_rate_granularity_seconds`: (default: `1`) granualarity of the aggregated requests for the request rate calculation. +- `request_rate_target_value`: (default: `100`) target value for the request rate. +- `request_rate_window_seconds`: (default: `60`) aggregation window for the request rate calculation. +- `concurrency_target_value`: (default: `100`) target value for the request concurrency. + +Note that due to Keda limitations, scaling is done by _either_ `concurrency` or `request_rate`. + #### Workload Autoscaling Authentication Most Keda triggers require some form of authentication to query for data. In the Kubernetes API, they are represented by `TriggerAuthentication` and `ClusterTriggerAuthentication` resources. Dokku can manage these via the `scheduler-k3s:autoscaling-auth` commands, and includes generated resources with each helm release generated by a deploy. diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index e369f2b3d..449103c2c 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,4 +1,4 @@ -SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall +SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index c1effe1d2..4f84edca3 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -584,8 +584,18 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { "DEPLOYMENT_NAME": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), } + validHttpScaleMethods := map[string]bool{ + "request_rate": true, + "concurrency": true, + } + + hasHttpTrigger := false + httpTrigger := ProcessAutoscalingTrigger{} triggers := []ProcessAutoscalingTrigger{} - for _, trigger := range config.Triggers { + for idx, trigger := range config.Triggers { + if trigger.Type == "" { + return ProcessAutoscaling{}, fmt.Errorf("Autoscaling trigger type is required for trigger: index %d", idx) + } metadata := map[string]string{} for key, value := range trigger.Metadata { tmpl, err := template.New("").Delims("[[", "]]").Parse(value) @@ -600,33 +610,77 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { metadata[key] = output.String() } - trigger := ProcessAutoscalingTrigger{ + if trigger.Name == "" { + trigger.Name = fmt.Sprintf("trigger-%s-%d", trigger.Type, idx+1) + } + + autoscalingTrigger := ProcessAutoscalingTrigger{ Name: trigger.Name, Type: trigger.Type, Metadata: metadata, } if auth, ok := input.KedaValues.Authentications[trigger.Type]; ok { - trigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ + autoscalingTrigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ Name: auth.Name, Kind: string(auth.Kind), } } else if auth, ok := input.KedaValues.GlobalAuthentications[trigger.Type]; ok { - trigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ + autoscalingTrigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ Name: auth.Name, Kind: string(auth.Kind), } } - triggers = append(triggers, trigger) + if autoscalingTrigger.Type == "http" { + if hasHttpTrigger { + return ProcessAutoscaling{}, errors.New("Only one http trigger is allowed") + } + + hasHttpTrigger = true + httpTrigger = autoscalingTrigger + + if _, ok := metadata["scale_by"]; !ok { + httpTrigger.Metadata["scale_by"] = "request_rate" + } + + if !validHttpScaleMethods[metadata["scale_by"]] { + return ProcessAutoscaling{}, fmt.Errorf("Invalid http scale method: %s", metadata["scale_by"]) + } + + if _, ok := metadata["scaledown_period_seconds"]; !ok { + httpTrigger.Metadata["scaledown_period_seconds"] = "300" + } + + if _, ok := metadata["request_rate_granularity_seconds"]; !ok { + httpTrigger.Metadata["request_rate_granularity_seconds"] = "1" + } + + if _, ok := metadata["request_rate_target_value"]; !ok { + httpTrigger.Metadata["request_rate_target_value"] = "100" + } + + if _, ok := metadata["request_rate_window_seconds"]; !ok { + httpTrigger.Metadata["request_rate_window_seconds"] = "60" + } + + if _, ok := metadata["concurrency_target_value"]; !ok { + httpTrigger.Metadata["concurrency_target_value"] = "100" + } + + continue + } + + triggers = append(triggers, autoscalingTrigger) } autoscaling := ProcessAutoscaling{ CooldownPeriodSeconds: ptr.Deref(config.CooldownPeriodSeconds, 300), - Enabled: len(triggers) > 0, + Enabled: len(triggers) > 0 || hasHttpTrigger, MaxReplicas: ptr.Deref(config.MaxQuantity, 0), MinReplicas: ptr.Deref(config.MinQuantity, 0), PollingIntervalSeconds: ptr.Deref(config.PollingIntervalSeconds, 30), + HttpTrigger: httpTrigger, Triggers: triggers, Type: "keda", } diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 098bf3ad1..8f007cdf1 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -21,6 +21,7 @@ import ( "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/kube" + "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/storage/driver" ) @@ -58,9 +59,12 @@ type ChartInput struct { } type Release struct { - Name string - Namespace string - Version int + AppVersion string + Name string + Namespace string + Revision int + Status release.Status + Version string } type HelmAgent struct { @@ -269,24 +273,61 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { return nil } -func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Release, error) { - client := action.NewHistory(h.Configuration) - response, err := client.Run(releaseName) +func (h *HelmAgent) InstalledRevision(releaseName string) (Release, error) { + revisions, err := h.ListRevisions(ListRevisionsInput{ + ReleaseName: releaseName, + Max: 1, + }) if err != nil { - return nil, fmt.Errorf("Error getting revisions: %w", err) + return Release{}, err + } + + if len(revisions) == 0 { + return Release{}, nil + } + + return revisions[0], nil +} + +type ListRevisionsInput struct { + ReleaseName string + Max int +} + +func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) { + client := action.NewHistory(h.Configuration) + if input.Max > 0 { + client.Max = input.Max } releases := []Release{} + response, err := client.Run(input.ReleaseName) + if err != nil { + if errors.Is(err, driver.ErrReleaseNotFound) { + return releases, nil + } + + return nil, fmt.Errorf("Error getting revisions: %w", err) + } + for _, release := range response { + appVersion := "MISSING" + if release.Chart != nil && release.Chart.Metadata != nil { + appVersion = release.Chart.AppVersion() + } + releases = append(releases, Release{ - Name: release.Name, - Namespace: release.Namespace, - Version: release.Version, + AppVersion: appVersion, + Name: release.Name, + Namespace: release.Namespace, + Revision: release.Version, + Status: release.Info.Status, + Version: release.Chart.Metadata.Version, }) } sort.Slice(releases, func(i, j int) bool { - return releases[i].Version < releases[j].Version + return releases[i].Revision < releases[j].Revision }) return releases, nil @@ -334,7 +375,8 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { client.RepoURL = input.RepoURL } - chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil) + settings := cli.New() + chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, settings) if err != nil { return fmt.Errorf("Error locating chart: %w", err) } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index d72005b22..b551cd574 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -122,6 +122,14 @@ var HelmCharts = []HelmChart{ RepoURL: "https://kedacore.github.io/charts", Version: "2.13.1", }, + { + ChartPath: "keda-add-ons-http", + CreateNamespace: true, + Namespace: "keda", + ReleaseName: "keda-add-ons-http", + RepoURL: "https://kedacore.github.io/charts", + Version: "0.8.0", + }, } type HelmRepository struct { diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index 702c3ba96..a87f59ad8 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -22,6 +22,7 @@ Additional commands:` scheduler-k3s:cluster-add [--insecure-allow-unknown-hosts] [--server-ip SERVER_IP] [--taint-scheduling] , Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list [--format json|stdout], Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id], Removes client node to a Dokku-managed cluster + scheduler-k3s:ensure-charts, Ensures the k3s charts are installed scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination scheduler-k3s:report [] [], Displays a scheduler-k3s report for one or more apps diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 883dc4a56..83c9e0db3 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -73,6 +73,10 @@ func main() { args.Parse(os.Args[2:]) nodeName := args.Arg(0) err = scheduler_k3s.CommandClusterRemove(nodeName) + case "ensure-charts": + args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError) + args.Parse(os.Args[2:]) + err = scheduler_k3s.CommandEnsureCharts() case "initialize": args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index ac22af732..51468339c 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -798,6 +798,91 @@ func CommandClusterRemove(nodeName string) error { return nil } +// CommandEnsureCharts ensures that the required helm charts are installed +func CommandEnsureCharts() error { + ctx, cancel := context.WithCancel(context.Background()) + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGQUIT, + syscall.SIGTERM) + go func() { + <-signals + cancel() + }() + + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Unable to create kubernetes client: %w", err) + } + + ingressClass := getGlobalIngressClass() + + namespacedHelmAgents := map[string]*HelmAgent{} + for _, chart := range HelmCharts { + _, ok := namespacedHelmAgents[chart.Namespace] + if !ok { + helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter) + if err != nil { + common.LogWarn(fmt.Sprintf("Unable to create helm agent: %s", err.Error())) + return err + } + namespacedHelmAgents[chart.Namespace] = helmAgent + } + } + + common.LogInfo2Quiet("Installing helm charts") + err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool { + common.LogInfo1(fmt.Sprintf("Processing chart %s@%s", chart.ReleaseName, chart.Version)) + if chart.ChartPath == "traefik" && ingressClass == "nginx" { + common.LogVerbose("Skipping chart due to ingress-class mismatch") + return false + } + + if chart.ChartPath == "ingress-nginx" && ingressClass == "traefik" { + common.LogVerbose("Skipping chart due to ingress-class mismatch") + return false + } + + helmAgent := namespacedHelmAgents[chart.Namespace] + latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName) + if err != nil { + common.LogWarn(fmt.Sprintf("Unable to get installed revision: %s", err)) + return false + } + + if latestRevision.Name == "" { + common.LogVerbose("Installing missing chart") + return true + } + + if latestRevision.Version != chart.Version { + common.LogVerbose(fmt.Sprintf("Installing chart due to version mismatch: %s != %s", latestRevision.AppVersion, chart.Version)) + return false + } + + common.LogVerbose("Skipping chart: already installed") + return false + }) + if err != nil { + return fmt.Errorf("Unable to install helm charts: %w", err) + } + + for _, manifest := range KubernetesManifests { + common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version)) + err = clientset.ApplyKubernetesManifest(ctx, ApplyKubernetesManifestInput{ + Manifest: manifest.Path, + }) + if err != nil { + return fmt.Errorf("Unable to apply kubernetes manifest: %w", err) + } + } + + common.LogInfo2Quiet("Done") + + return nil +} + // CommandLabelsSet set or clear a scheduler-k3s label for an app func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error { if resourceType == "" { diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index abc628ab3..17b5acaa5 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -61,8 +61,14 @@ type GlobalImage struct { } type GlobalNetwork struct { + // IngressClass is the default ingress class to use IngressClass string `yaml:"ingress_class"` - PrimaryPort int32 `yaml:"primary_port"` + + // PrimaryPort is the primary port to use + PrimaryPort int32 `yaml:"primary_port"` + + // PrimaryServicePort is the primary service port to use + PrimaryServicePort int32 `yaml:"primary_service_port"` } // GlobalKedaValues contains the global keda configuration @@ -116,6 +122,8 @@ type ProcessAnnotations struct { IngressAnnotations map[string]string `yaml:"ingress,omitempty"` JobAnnotations map[string]string `yaml:"job,omitempty"` KedaScalingObjectAnnotations map[string]string `yaml:"keda_scaled_object,omitempty"` + KedaHTTPScaledObjectAnnotations map[string]string `yaml:"keda_http_scaled_object,omitempty"` + KedaInterceptorProxyAnnotations map[string]string `yaml:"keda_interceptor_proxy,omitempty"` KedaSecretAnnotations map[string]string `yaml:"keda_secret,omitempty"` KedaTriggerAuthenticationAnnotations map[string]string `yaml:"keda_trigger_authentication,omitempty"` PodAnnotations map[string]string `yaml:"pod,omitempty"` @@ -143,6 +151,9 @@ type ProcessAutoscaling struct { // PollingIntervalSeconds is the number of seconds between polling for new metrics PollingIntervalSeconds int `yaml:"polling_interval_seconds,omitempty"` + // HttpTrigger is the http trigger config to use for autoscaling + HttpTrigger ProcessAutoscalingTrigger `yaml:"http_trigger,omitempty"` + // Triggers is a list of triggers to use for autoscaling Triggers []ProcessAutoscalingTrigger `yaml:"triggers,omitempty"` @@ -224,17 +235,22 @@ const ( ) type ProcessLabels struct { - CertificateLabels map[string]string `yaml:"certificate,omitempty"` - CronJobLabels map[string]string `yaml:"cronjob,omitempty"` - DeploymentLabels map[string]string `yaml:"deployment,omitempty"` - IngressLabels map[string]string `yaml:"ingress,omitempty"` - JobLabels map[string]string `yaml:"job,omitempty"` - PodLabels map[string]string `yaml:"pod,omitempty"` - SecretLabels map[string]string `yaml:"secret,omitempty"` - ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` - ServiceLabels map[string]string `yaml:"service,omitempty"` - TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` - TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` + CertificateLabels map[string]string `yaml:"certificate,omitempty"` + CronJobLabels map[string]string `yaml:"cronjob,omitempty"` + DeploymentLabels map[string]string `yaml:"deployment,omitempty"` + IngressLabels map[string]string `yaml:"ingress,omitempty"` + JobLabels map[string]string `yaml:"job,omitempty"` + KedaScalingObjectLabels map[string]string `yaml:"keda_scaled_object,omitempty"` + KedaHTTPScaledObjectLabels map[string]string `yaml:"keda_http_scaled_object,omitempty"` + KedaInterceptorProxyLabels map[string]string `yaml:"keda_interceptor_proxy,omitempty"` + KedaSecretLabels map[string]string `yaml:"keda_secret,omitempty"` + KedaTriggerAuthenticationLabels map[string]string `yaml:"keda_trigger_authentication,omitempty"` + PodLabels map[string]string `yaml:"pod,omitempty"` + SecretLabels map[string]string `yaml:"secret,omitempty"` + ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` + ServiceLabels map[string]string `yaml:"service,omitempty"` + TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` + TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` } type ProcessWeb struct { diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index b0eaa6c57..5530525f3 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,5 +1,11 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} +{{- $routeToKeda := false }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }} +{{- $routeToKeda = true }} +{{- end }} +{{- end }} {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} {{- range $pdx, $domain := $config.web.domains }} --- @@ -36,11 +42,19 @@ spec: paths: {{- if $config.web.port_maps }} - backend: + {{- if $routeToKeda}} + service: + name: kip-{{ $.Values.global.app_name}}-{{ $processName }} + port: + number: 8080 + pathType: Prefix + {{- else }} service: name: {{ $.Values.global.app_name }}-{{ $processName }} port: name: {{ include "primary.port" $config.web.port_maps }} pathType: ImplementationSpecific + {{- end }} path: / {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml new file mode 100644 index 000000000..bd761642f --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -0,0 +1,53 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := index .Values.processes "PROCESS_NAME" }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }} +{{- $mappings := dict }} +{{- range $pdx, $port_map := $config.web.port_maps }} +{{- $mappings := set $mappings $port_map.name "true" }} +{{- end }} +--- +kind: HTTPScaledObject +apiVersion: http.keda.sh/v1alpha1 +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} + dokku.com/managed: "true" + httpscaledobject.keda.sh/skip-scaledobject-creation: "true" + {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }} + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }} + name: {{ $.Values.global.app_name}}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} +spec: + hosts: + {{- range $pdx, $domain := $config.web.domains }} + - {{ $domain.name | quote }} + {{- end }} + scaleTargetRef: + name: {{ $.Values.global.app_name }}-{{ $processName }} + kind: Deployment + apiVersion: apps/v1 + service: {{ $.Values.global.app_name }}-{{ $processName }} + port: {{ $.Values.global.network.primary_service_port }} + replicas: + min: {{ $config.autoscaling.min_replicas }} + max: {{ $config.autoscaling.max_replicas }} + scaledownPeriod: {{ get $config.autoscaling.http_trigger.metadata "scaledown_period_seconds" }} + scalingMetric: + {{- if eq (get $config.autoscaling.http_trigger.metadata "scale_by") "request_rate" }} + requestRate: + granularity: {{ get $config.autoscaling.http_trigger.metadata "request_rate_granularity_seconds" }}s + targetValue: {{ get $config.autoscaling.http_trigger.metadata "request_rate_target_value" }} + window: {{ get $config.autoscaling.http_trigger.metadata "request_rate_window_seconds" }}s + {{- else }} + concurrency: + targetValue: {{ get $config.autoscaling.http_trigger.metadata "concurrency_target_value" }} + {{- end }} +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml new file mode 100644 index 000000000..c12d6411e --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml @@ -0,0 +1,26 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := index .Values.processes "PROCESS_NAME" }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} +--- +kind: Service +apiVersion: v1 +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} + dokku.com/managed: "true" + {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }} + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }} + name: kip-{{ $.Values.global.app_name}}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} +spec: + type: ExternalName + externalName: keda-add-ons-http-interceptor-proxy.keda.svc.cluster.local +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml index 0cb037d94..24e6ed5e1 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml @@ -2,6 +2,7 @@ {{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}} +--- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: @@ -31,9 +32,13 @@ spec: fallback: failureThreshold: 3 replicas: {{ $config.replicas }} + {{- if $config.autoscaling.triggers }} triggers: {{- with $config.autoscaling.triggers }} {{- toYaml . | nindent 4 }} {{- end }} + {{- else }} + triggers: [] + {{- end }} {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml b/plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml new file mode 100644 index 000000000..3ad4da833 --- /dev/null +++ b/plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml @@ -0,0 +1,31 @@ +--- +operator: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule +scaler: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule +interceptor: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 1edafd3ae..6763d1ab8 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -250,8 +250,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } primaryPort := int32(5000) + primaryServicePort := int32(80) for _, portMap := range portMaps { primaryPort = portMap.ContainerPort + primaryServicePort = portMap.HostPort if primaryPort != 0 { break } @@ -350,8 +352,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Labels: globalLabels, Namespace: namespace, Network: GlobalNetwork{ - IngressClass: getGlobalIngressClass(), - PrimaryPort: primaryPort, + IngressClass: getGlobalIngressClass(), + PrimaryPort: primaryPort, + PrimaryServicePort: primaryServicePort, }, Secrets: map[string]string{}, }, @@ -504,7 +507,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e templateFiles := []string{"deployment", "keda-scaled-object"} if processType == "web" { - templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware") + templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service") } for _, templateName := range templateFiles { b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) diff --git a/tests/unit/git_3.bats b/tests/unit/git_3.bats index bfc3a386e..a86b2e09c 100644 --- a/tests/unit/git_3.bats +++ b/tests/unit/git_3.bats @@ -5,7 +5,7 @@ load test_helper SMOKE_TEST_APP_1_0_0_SHA=9cf71bba639c4f1671dfd42685338b762d3354f2 SMOKE_TEST_APP_2_0_0_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9 SMOKE_TEST_APP_ANOTHER_BRANCH_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9 -SMOKE_TEST_APP_MASTER_SHA=af1b02052199b8ca8115f80ff8676a7c7744a45f +SMOKE_TEST_APP_MASTER_SHA=96f6ba2db912c93707080a358016d1febb7c560a SMOKE_TEST_APP_COMMIT_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9 setup() {