From 69f94f64f0978973c5f9d9078019dd54b6520352 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 01:17:27 -0500 Subject: [PATCH 01/13] fix: correct issue with locating existing chart --- plugins/scheduler-k3s/helm.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 098bf3ad1..719346d45 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -334,7 +334,8 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { client.RepoURL = input.RepoURL } - chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil) + settings := cli.New() + chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, settings) if err != nil { return fmt.Errorf("Error locating chart: %w", err) } From 8aa311131e4d62eb8364327b0f31a541d215c46b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 01:18:40 -0500 Subject: [PATCH 02/13] feat: add scheduler-k3s:ensure-charts This command can be used to ensure charts are properly installed and configured according to Dokku's internal settings. --- docs/deployment/schedulers/k3s.md | 1 + plugins/scheduler-k3s/Makefile | 2 +- plugins/scheduler-k3s/helm.go | 63 ++++++++++--- .../scheduler-k3s/src/commands/commands.go | 1 + .../src/subcommands/subcommands.go | 4 + plugins/scheduler-k3s/subcommands.go | 90 +++++++++++++++++++ 6 files changed, 149 insertions(+), 12 deletions(-) diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index f1c66cee5..c1c4e705c 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -10,6 +10,7 @@ scheduler-k3s:autoscaling-auth:report [--format stdout|json] [--i scheduler-k3s:cluster-add [ssh://user@host:port] # Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list # Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id] # Removes client node to a Dokku-managed cluster +scheduler-k3s:ensure-charts # Ensures the k3s charts are installed scheduler-k3s:initialize # Initializes a cluster scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination scheduler-k3s:report [] [] # Displays a scheduler-k3s report for one or more apps diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index e369f2b3d..449103c2c 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,4 +1,4 @@ -SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall +SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 719346d45..8f007cdf1 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -21,6 +21,7 @@ import ( "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/kube" + "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/storage/driver" ) @@ -58,9 +59,12 @@ type ChartInput struct { } type Release struct { - Name string - Namespace string - Version int + AppVersion string + Name string + Namespace string + Revision int + Status release.Status + Version string } type HelmAgent struct { @@ -269,24 +273,61 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { return nil } -func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Release, error) { - client := action.NewHistory(h.Configuration) - response, err := client.Run(releaseName) +func (h *HelmAgent) InstalledRevision(releaseName string) (Release, error) { + revisions, err := h.ListRevisions(ListRevisionsInput{ + ReleaseName: releaseName, + Max: 1, + }) if err != nil { - return nil, fmt.Errorf("Error getting revisions: %w", err) + return Release{}, err + } + + if len(revisions) == 0 { + return Release{}, nil + } + + return revisions[0], nil +} + +type ListRevisionsInput struct { + ReleaseName string + Max int +} + +func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) { + client := action.NewHistory(h.Configuration) + if input.Max > 0 { + client.Max = input.Max } releases := []Release{} + response, err := client.Run(input.ReleaseName) + if err != nil { + if errors.Is(err, driver.ErrReleaseNotFound) { + return releases, nil + } + + return nil, fmt.Errorf("Error getting revisions: %w", err) + } + for _, release := range response { + appVersion := "MISSING" + if release.Chart != nil && release.Chart.Metadata != nil { + appVersion = release.Chart.AppVersion() + } + releases = append(releases, Release{ - Name: release.Name, - Namespace: release.Namespace, - Version: release.Version, + AppVersion: appVersion, + Name: release.Name, + Namespace: release.Namespace, + Revision: release.Version, + Status: release.Info.Status, + Version: release.Chart.Metadata.Version, }) } sort.Slice(releases, func(i, j int) bool { - return releases[i].Version < releases[j].Version + return releases[i].Revision < releases[j].Revision }) return releases, nil diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index 702c3ba96..a87f59ad8 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -22,6 +22,7 @@ Additional commands:` scheduler-k3s:cluster-add [--insecure-allow-unknown-hosts] [--server-ip SERVER_IP] [--taint-scheduling] , Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list [--format json|stdout], Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id], Removes client node to a Dokku-managed cluster + scheduler-k3s:ensure-charts, Ensures the k3s charts are installed scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination scheduler-k3s:report [] [], Displays a scheduler-k3s report for one or more apps diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 883dc4a56..83c9e0db3 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -73,6 +73,10 @@ func main() { args.Parse(os.Args[2:]) nodeName := args.Arg(0) err = scheduler_k3s.CommandClusterRemove(nodeName) + case "ensure-charts": + args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError) + args.Parse(os.Args[2:]) + err = scheduler_k3s.CommandEnsureCharts() case "initialize": args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index ac22af732..5691209e3 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -798,6 +798,96 @@ func CommandClusterRemove(nodeName string) error { return nil } +// CommandEnsureCharts ensures that the required helm charts are installed +func CommandEnsureCharts() error { + ctx, cancel := context.WithCancel(context.Background()) + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGQUIT, + syscall.SIGTERM) + go func() { + <-signals + cancel() + }() + + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Unable to create kubernetes client: %w", err) + } + + ingressClass := getGlobalIngressClass() + + namespacedHelmAgents := map[string]*HelmAgent{} + for _, chart := range HelmCharts { + _, ok := namespacedHelmAgents[chart.Namespace] + if !ok { + helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter) + if err != nil { + common.LogWarn(fmt.Sprintf("Unable to create helm agent: %s", err.Error())) + return err + } + namespacedHelmAgents[chart.Namespace] = helmAgent + } + } + + common.LogInfo2Quiet("Installing helm charts") + err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool { + common.LogInfo1(fmt.Sprintf("Processing chart %s@%s", chart.ReleaseName, chart.Version)) + if chart.ChartPath == "traefik" && ingressClass == "nginx" { + common.LogVerbose("Skipping chart due to ingress-class mismatch") + return false + } + + if chart.ChartPath == "ingress-nginx" && ingressClass == "traefik" { + common.LogVerbose("Skipping chart due to ingress-class mismatch") + return false + } + + if chart.ReleaseName == "keda-add-ons-http" { + common.LogVerbose("Forcing install of keda-add-ons-http") + return true + } + + helmAgent := namespacedHelmAgents[chart.Namespace] + latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName) + if err != nil { + common.LogWarn(fmt.Sprintf("Unable to get installed revision: %s", err)) + return false + } + + if latestRevision.Name == "" { + common.LogVerbose("Installing missing chart") + return true + } + + if latestRevision.Version != chart.Version { + common.LogVerbose(fmt.Sprintf("Installing chart due to version mismatch: %s != %s", latestRevision.AppVersion, chart.Version)) + return false + } + + common.LogVerbose("Skipping chart: already installed") + return false + }) + if err != nil { + return fmt.Errorf("Unable to install helm charts: %w", err) + } + + for _, manifest := range KubernetesManifests { + common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version)) + err = clientset.ApplyKubernetesManifest(ctx, ApplyKubernetesManifestInput{ + Manifest: manifest.Path, + }) + if err != nil { + return fmt.Errorf("Unable to apply kubernetes manifest: %w", err) + } + } + + common.LogInfo2Quiet("Done") + + return nil +} + // CommandLabelsSet set or clear a scheduler-k3s label for an app func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error { if resourceType == "" { From f8d2768816c23ea0d2918fbdd552abf7ab29848b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 02:05:20 -0500 Subject: [PATCH 03/13] feat: default to no triggers on any created ScaledObject --- .../scheduler-k3s/templates/chart/keda-scaled-object.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml index 0cb037d94..24e6ed5e1 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml @@ -2,6 +2,7 @@ {{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }} {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}} +--- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: @@ -31,9 +32,13 @@ spec: fallback: failureThreshold: 3 replicas: {{ $config.replicas }} + {{- if $config.autoscaling.triggers }} triggers: {{- with $config.autoscaling.triggers }} {{- toYaml . | nindent 4 }} {{- end }} + {{- else }} + triggers: [] + {{- end }} {{- end }} {{- end }} From 785af1611ca7840a2cfa18c8a7f1a370f4e2bb91 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 02:14:29 -0500 Subject: [PATCH 04/13] fix: error if there is no type for the trigger specified --- plugins/scheduler-k3s/functions.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index c1effe1d2..4913c8a3b 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -586,6 +586,9 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { triggers := []ProcessAutoscalingTrigger{} for _, trigger := range config.Triggers { + if trigger.Type == "" { + return ProcessAutoscaling{}, fmt.Errorf("Autoscaling trigger type is required for trigger: index %d", idx) + } metadata := map[string]string{} for key, value := range trigger.Metadata { tmpl, err := template.New("").Delims("[[", "]]").Parse(value) From 62f25ad33fe416a6a31d687e7f44603568c4f2a8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 02:14:54 -0500 Subject: [PATCH 05/13] feat: give the trigger a default name if none is specified --- plugins/scheduler-k3s/functions.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 4913c8a3b..9c6c34b0c 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -585,7 +585,7 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { } triggers := []ProcessAutoscalingTrigger{} - for _, trigger := range config.Triggers { + for idx, trigger := range config.Triggers { if trigger.Type == "" { return ProcessAutoscaling{}, fmt.Errorf("Autoscaling trigger type is required for trigger: index %d", idx) } @@ -603,7 +603,9 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { metadata[key] = output.String() } - trigger := ProcessAutoscalingTrigger{ + if trigger.Name == "" { + trigger.Name = fmt.Sprintf("trigger-%s-%d", trigger.Type, idx+1) + } Name: trigger.Name, Type: trigger.Type, Metadata: metadata, From 7e406436fa0064083381a40e0ef2fb7e11ae45c0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 02:15:18 -0500 Subject: [PATCH 06/13] fix: do not shadow the trigger variable --- plugins/scheduler-k3s/functions.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 9c6c34b0c..e3b0f42ca 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -606,24 +606,26 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { if trigger.Name == "" { trigger.Name = fmt.Sprintf("trigger-%s-%d", trigger.Type, idx+1) } + + autoscalingTrigger := ProcessAutoscalingTrigger{ Name: trigger.Name, Type: trigger.Type, Metadata: metadata, } if auth, ok := input.KedaValues.Authentications[trigger.Type]; ok { - trigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ + autoscalingTrigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ Name: auth.Name, Kind: string(auth.Kind), } } else if auth, ok := input.KedaValues.GlobalAuthentications[trigger.Type]; ok { - trigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ + autoscalingTrigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{ Name: auth.Name, Kind: string(auth.Kind), } } - triggers = append(triggers, trigger) + triggers = append(triggers, autoscalingTrigger) } autoscaling := ProcessAutoscaling{ From 1751bf30ba3d35b6c3c9e26b30f6e5ee6439a6e9 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 02:15:47 -0500 Subject: [PATCH 07/13] feat: install the keda-http-add-on --- plugins/scheduler-k3s/scheduler_k3s.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index d72005b22..b551cd574 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -122,6 +122,14 @@ var HelmCharts = []HelmChart{ RepoURL: "https://kedacore.github.io/charts", Version: "2.13.1", }, + { + ChartPath: "keda-add-ons-http", + CreateNamespace: true, + Namespace: "keda", + ReleaseName: "keda-add-ons-http", + RepoURL: "https://kedacore.github.io/charts", + Version: "0.8.0", + }, } type HelmRepository struct { From b98863b6a7bb8c80251fea50cc431b126685321f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 03:29:26 -0500 Subject: [PATCH 08/13] feat: add support for the keda-http-add-on Externally, it is treated the same as any other scaling trigger, but requires that the chart be installed in order for it to be supported. --- docs/deployment/schedulers/k3s.md | 50 +++++++++++++++++ plugins/scheduler-k3s/functions.go | 49 ++++++++++++++++- plugins/scheduler-k3s/template.go | 40 ++++++++++---- .../templates/chart/ingress.yaml | 14 +++++ .../chart/keda-http-scaled-object.yaml | 55 +++++++++++++++++++ .../chart/keda-interceptor-proxy-service.yaml | 26 +++++++++ plugins/scheduler-k3s/triggers.go | 9 ++- 7 files changed, 227 insertions(+), 16 deletions(-) create mode 100644 plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml create mode 100644 plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index c1c4e705c..873406b5a 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -417,6 +417,56 @@ Each value in the `metadata` stanza can use the following interpolated strings: - `DOKKU_PROCESS_TYPE`: The name of the process being scaled - `DOKKU_APP_NAME`: The name of the app being scaled +##### HTTP Autoscaling + +In addition to the built-in scalers that Keda provides, Dokku also supports Keda's HTTP Add On. This requires that the addon be properly installed and configured. For existing k3s clusters, this can be performed by the `scheduler-k3s:ensure-charts` command: + +```shell +dokku scheduler-k3s:ensure-charts +``` + +> [!NOTE] +> Users who wish to use this functionality on a cluster not managed by Dokku will need to manually install the `keda-http-add-on` into the `keda` namespace. Please consult the `keda-http-add-on` [install documentation](https://kedacore.github.io/http-add-on/install.html) for further details. + +> [!WARNING] +> If the `keda-http-add-on` chart is not installed, then this trigger will be ignored. + +Once the chart is configured, an `http` trigger can be specified like so: + + +```json +{ + "formation": { + "web": { + "autoscaling": { + "min_quantity": 1, + "max_quantity": 10, + "triggers": [ + { + "type": "http", + "metadata": { + "scaledown_period_seconds": "150", + "request_rate_target_value": "50" + } + } + ] + } + } + } +} +``` + +The following metadata properties are supported with the http autoscaler: + +- `scale_by`: (default: `request_rate`) whether to scale by `concurrency` or `request_rate`. +- `scaledown_period_seconds`: (default: `300`) period to wait after the last reported active before scaling the resource back to 0. +- `request_rate_granularity_seconds`: (default: `1`) granualarity of the aggregated requests for the request rate calculation. +- `request_rate_target_value`: (default: `100`) target value for the request rate. +- `request_rate_window_seconds`: (default: `60`) aggregation window for the request rate calculation. +- `concurrency_target_value`: (default: `100`) target value for the request concurrency. + +Note that due to Keda limitations, scaling is done by _either_ `concurrency` or `request_rate`. + #### Workload Autoscaling Authentication Most Keda triggers require some form of authentication to query for data. In the Kubernetes API, they are represented by `TriggerAuthentication` and `ClusterTriggerAuthentication` resources. Dokku can manage these via the `scheduler-k3s:autoscaling-auth` commands, and includes generated resources with each helm release generated by a deploy. diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index e3b0f42ca..4f84edca3 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -584,6 +584,13 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { "DEPLOYMENT_NAME": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), } + validHttpScaleMethods := map[string]bool{ + "request_rate": true, + "concurrency": true, + } + + hasHttpTrigger := false + httpTrigger := ProcessAutoscalingTrigger{} triggers := []ProcessAutoscalingTrigger{} for idx, trigger := range config.Triggers { if trigger.Type == "" { @@ -625,15 +632,55 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) { } } + if autoscalingTrigger.Type == "http" { + if hasHttpTrigger { + return ProcessAutoscaling{}, errors.New("Only one http trigger is allowed") + } + + hasHttpTrigger = true + httpTrigger = autoscalingTrigger + + if _, ok := metadata["scale_by"]; !ok { + httpTrigger.Metadata["scale_by"] = "request_rate" + } + + if !validHttpScaleMethods[metadata["scale_by"]] { + return ProcessAutoscaling{}, fmt.Errorf("Invalid http scale method: %s", metadata["scale_by"]) + } + + if _, ok := metadata["scaledown_period_seconds"]; !ok { + httpTrigger.Metadata["scaledown_period_seconds"] = "300" + } + + if _, ok := metadata["request_rate_granularity_seconds"]; !ok { + httpTrigger.Metadata["request_rate_granularity_seconds"] = "1" + } + + if _, ok := metadata["request_rate_target_value"]; !ok { + httpTrigger.Metadata["request_rate_target_value"] = "100" + } + + if _, ok := metadata["request_rate_window_seconds"]; !ok { + httpTrigger.Metadata["request_rate_window_seconds"] = "60" + } + + if _, ok := metadata["concurrency_target_value"]; !ok { + httpTrigger.Metadata["concurrency_target_value"] = "100" + } + + continue + } + triggers = append(triggers, autoscalingTrigger) } autoscaling := ProcessAutoscaling{ CooldownPeriodSeconds: ptr.Deref(config.CooldownPeriodSeconds, 300), - Enabled: len(triggers) > 0, + Enabled: len(triggers) > 0 || hasHttpTrigger, MaxReplicas: ptr.Deref(config.MaxQuantity, 0), MinReplicas: ptr.Deref(config.MinQuantity, 0), PollingIntervalSeconds: ptr.Deref(config.PollingIntervalSeconds, 30), + HttpTrigger: httpTrigger, Triggers: triggers, Type: "keda", } diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index abc628ab3..17b5acaa5 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -61,8 +61,14 @@ type GlobalImage struct { } type GlobalNetwork struct { + // IngressClass is the default ingress class to use IngressClass string `yaml:"ingress_class"` - PrimaryPort int32 `yaml:"primary_port"` + + // PrimaryPort is the primary port to use + PrimaryPort int32 `yaml:"primary_port"` + + // PrimaryServicePort is the primary service port to use + PrimaryServicePort int32 `yaml:"primary_service_port"` } // GlobalKedaValues contains the global keda configuration @@ -116,6 +122,8 @@ type ProcessAnnotations struct { IngressAnnotations map[string]string `yaml:"ingress,omitempty"` JobAnnotations map[string]string `yaml:"job,omitempty"` KedaScalingObjectAnnotations map[string]string `yaml:"keda_scaled_object,omitempty"` + KedaHTTPScaledObjectAnnotations map[string]string `yaml:"keda_http_scaled_object,omitempty"` + KedaInterceptorProxyAnnotations map[string]string `yaml:"keda_interceptor_proxy,omitempty"` KedaSecretAnnotations map[string]string `yaml:"keda_secret,omitempty"` KedaTriggerAuthenticationAnnotations map[string]string `yaml:"keda_trigger_authentication,omitempty"` PodAnnotations map[string]string `yaml:"pod,omitempty"` @@ -143,6 +151,9 @@ type ProcessAutoscaling struct { // PollingIntervalSeconds is the number of seconds between polling for new metrics PollingIntervalSeconds int `yaml:"polling_interval_seconds,omitempty"` + // HttpTrigger is the http trigger config to use for autoscaling + HttpTrigger ProcessAutoscalingTrigger `yaml:"http_trigger,omitempty"` + // Triggers is a list of triggers to use for autoscaling Triggers []ProcessAutoscalingTrigger `yaml:"triggers,omitempty"` @@ -224,17 +235,22 @@ const ( ) type ProcessLabels struct { - CertificateLabels map[string]string `yaml:"certificate,omitempty"` - CronJobLabels map[string]string `yaml:"cronjob,omitempty"` - DeploymentLabels map[string]string `yaml:"deployment,omitempty"` - IngressLabels map[string]string `yaml:"ingress,omitempty"` - JobLabels map[string]string `yaml:"job,omitempty"` - PodLabels map[string]string `yaml:"pod,omitempty"` - SecretLabels map[string]string `yaml:"secret,omitempty"` - ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` - ServiceLabels map[string]string `yaml:"service,omitempty"` - TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` - TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` + CertificateLabels map[string]string `yaml:"certificate,omitempty"` + CronJobLabels map[string]string `yaml:"cronjob,omitempty"` + DeploymentLabels map[string]string `yaml:"deployment,omitempty"` + IngressLabels map[string]string `yaml:"ingress,omitempty"` + JobLabels map[string]string `yaml:"job,omitempty"` + KedaScalingObjectLabels map[string]string `yaml:"keda_scaled_object,omitempty"` + KedaHTTPScaledObjectLabels map[string]string `yaml:"keda_http_scaled_object,omitempty"` + KedaInterceptorProxyLabels map[string]string `yaml:"keda_interceptor_proxy,omitempty"` + KedaSecretLabels map[string]string `yaml:"keda_secret,omitempty"` + KedaTriggerAuthenticationLabels map[string]string `yaml:"keda_trigger_authentication,omitempty"` + PodLabels map[string]string `yaml:"pod,omitempty"` + SecretLabels map[string]string `yaml:"secret,omitempty"` + ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"` + ServiceLabels map[string]string `yaml:"service,omitempty"` + TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"` + TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"` } type ProcessWeb struct { diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index b0eaa6c57..fbe512c17 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,5 +1,11 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} +{{- $routeToKeda := false }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") $config.autoscaling.http_trigger.type }} +{{- $routeToKeda = true }} +{{- end }} +{{- end }} {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} {{- range $pdx, $domain := $config.web.domains }} --- @@ -36,11 +42,19 @@ spec: paths: {{- if $config.web.port_maps }} - backend: + {{- if $routeToKeda}} + service: + name: kip-{{ $.Values.global.app_name}}-{{ $processName }} + port: + number: 8080 + pathType: Prefix + {{- else }} service: name: {{ $.Values.global.app_name }}-{{ $processName }} port: name: {{ include "primary.port" $config.web.port_maps }} pathType: ImplementationSpecific + {{- end }} path: / {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml new file mode 100644 index 000000000..e0a12c70b --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -0,0 +1,55 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := index .Values.processes "PROCESS_NAME" }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} +{{- if $config.autoscaling.http_trigger.type }} +{{- $mappings := dict }} +{{- range $pdx, $port_map := $config.web.port_maps }} +{{- $mappings := set $mappings $port_map.name "true" }} +{{- end }} +--- +kind: HTTPScaledObject +apiVersion: http.keda.sh/v1alpha1 +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} + dokku.com/managed: "true" + httpscaledobject.keda.sh/skip-scaledobject-creation: "true" + {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }} + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }} + name: {{ $.Values.global.app_name}}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} +spec: + hosts: + {{- range $pdx, $domain := $config.web.domains }} + - {{ $domain.name | quote }} + {{- end }} + scaleTargetRef: + name: {{ $.Values.global.app_name }}-{{ $processName }} + kind: Deployment + apiVersion: apps/v1 + service: {{ $.Values.global.app_name }}-{{ $processName }} + port: {{ $.Values.global.network.primary_service_port }} + replicas: + min: {{ $config.autoscaling.min_replicas }} + max: {{ $config.autoscaling.max_replicas }} + scaledownPeriod: {{ get $config.autoscaling.http_trigger.metadata "scaledown_period_seconds" }} + scalingMetric: + {{- if eq (get $config.autoscaling.http_trigger.metadata "scale_by") "request_rate" }} + requestRate: + granularity: {{ get $config.autoscaling.http_trigger.metadata "request_rate_granularity_seconds" }}s + targetValue: {{ get $config.autoscaling.http_trigger.metadata "request_rate_target_value" }} + window: {{ get $config.autoscaling.http_trigger.metadata "request_rate_window_seconds" }}s + {{- else }} + concurrency: + targetValue: {{ get $config.autoscaling.http_trigger.metadata "concurrency_target_value" }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml new file mode 100644 index 000000000..c12d6411e --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml @@ -0,0 +1,26 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := index .Values.processes "PROCESS_NAME" }} +{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} +{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} +--- +kind: Service +apiVersion: v1 +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} + dokku.com/managed: "true" + {{ include "print.annotations" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }} + {{ include "print.annotations" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }} + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + {{ include "print.labels" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }} + {{ include "print.labels" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }} + name: kip-{{ $.Values.global.app_name}}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} +spec: + type: ExternalName + externalName: keda-add-ons-http-interceptor-proxy.keda.svc.cluster.local +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 1edafd3ae..6763d1ab8 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -250,8 +250,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } primaryPort := int32(5000) + primaryServicePort := int32(80) for _, portMap := range portMaps { primaryPort = portMap.ContainerPort + primaryServicePort = portMap.HostPort if primaryPort != 0 { break } @@ -350,8 +352,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Labels: globalLabels, Namespace: namespace, Network: GlobalNetwork{ - IngressClass: getGlobalIngressClass(), - PrimaryPort: primaryPort, + IngressClass: getGlobalIngressClass(), + PrimaryPort: primaryPort, + PrimaryServicePort: primaryServicePort, }, Secrets: map[string]string{}, }, @@ -504,7 +507,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e templateFiles := []string{"deployment", "keda-scaled-object"} if processType == "web" { - templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware") + templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service") } for _, templateName := range templateFiles { b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) From f2a4fe493aab521d75de339136051fe73f074d45 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 03:29:54 -0500 Subject: [PATCH 09/13] fix: do not force-reinstall the keda-http-add-on --- plugins/scheduler-k3s/subcommands.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 5691209e3..51468339c 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -844,11 +844,6 @@ func CommandEnsureCharts() error { return false } - if chart.ReleaseName == "keda-add-ons-http" { - common.LogVerbose("Forcing install of keda-add-ons-http") - return true - } - helmAgent := namespacedHelmAgents[chart.Namespace] latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName) if err != nil { From 52f08ff793b46f20ef57f26a82fc41ec64377386 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 04:06:37 -0500 Subject: [PATCH 10/13] fix: do not perform nil lookup --- .../scheduler-k3s/templates/chart/keda-http-scaled-object.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml index e0a12c70b..92a26edb4 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -2,7 +2,7 @@ {{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} {{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} -{{- if $config.autoscaling.http_trigger.type }} +{{- if and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http") }} {{- $mappings := dict }} {{- range $pdx, $port_map := $config.web.port_maps }} {{- $mappings := set $mappings $port_map.name "true" }} From 6cb97e75c90bee62e65e2b14c61c21100a5a31ee Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 04:06:58 -0500 Subject: [PATCH 11/13] fix: update the master commit for the smoke-test-app repo --- tests/unit/git_3.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/git_3.bats b/tests/unit/git_3.bats index bfc3a386e..a86b2e09c 100644 --- a/tests/unit/git_3.bats +++ b/tests/unit/git_3.bats @@ -5,7 +5,7 @@ load test_helper SMOKE_TEST_APP_1_0_0_SHA=9cf71bba639c4f1671dfd42685338b762d3354f2 SMOKE_TEST_APP_2_0_0_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9 SMOKE_TEST_APP_ANOTHER_BRANCH_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9 -SMOKE_TEST_APP_MASTER_SHA=af1b02052199b8ca8115f80ff8676a7c7744a45f +SMOKE_TEST_APP_MASTER_SHA=96f6ba2db912c93707080a358016d1febb7c560a SMOKE_TEST_APP_COMMIT_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9 setup() { From 44eab143ab7491df87a39fe6aa56007ac33ad27e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 17:10:41 -0500 Subject: [PATCH 12/13] fix: do not perform nil lookup --- plugins/scheduler-k3s/templates/chart/ingress.yaml | 2 +- .../templates/chart/keda-http-scaled-object.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index fbe512c17..5530525f3 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -2,7 +2,7 @@ {{- $config := index .Values.processes "PROCESS_NAME" }} {{- $routeToKeda := false }} {{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} -{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") $config.autoscaling.http_trigger.type }} +{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }} {{- $routeToKeda = true }} {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml index 92a26edb4..bd761642f 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml @@ -1,8 +1,7 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }} -{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}} -{{- if and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http") }} +{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }} {{- $mappings := dict }} {{- range $pdx, $port_map := $config.web.port_maps }} {{- $mappings := set $mappings $port_map.name "true" }} @@ -52,4 +51,3 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} From dcdbe22d863032ebdeb3dde5f417ef426e7f8302 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 13 Nov 2024 20:13:38 -0500 Subject: [PATCH 13/13] fix: add tolerations for keda --- .../helm-config/keda-add-ons-http.yaml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml diff --git a/plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml b/plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml new file mode 100644 index 000000000..3ad4da833 --- /dev/null +++ b/plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml @@ -0,0 +1,31 @@ +--- +operator: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule +scaler: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule +interceptor: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule