github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #7355 from dokku/k3s-keda-http-add-on

Browse Source
This commit is contained in:
Jose Diaz-Gonzalez
2024-11-14 01:32:59 -05:00
committed by GitHub
parent 4c0f415bad dcdbe22d86
commit d63e05f4ef
16 changed files with 428 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
docs/deployment/schedulers/k3s.md
View File

@@ -10,6 +10,7 @@ scheduler-k3s:autoscaling-auth:report <app|--global> [--format stdout|json] [--i
scheduler-k3s:cluster-add [ssh://user@host:port] # Adds a server node to a Dokku-managed cluster
scheduler-k3s:cluster-list # Lists all nodes in a Dokku-managed cluster
scheduler-k3s:cluster-remove [node-id] # Removes client node to a Dokku-managed cluster
scheduler-k3s:ensure-charts # Ensures the k3s charts are installed
scheduler-k3s:initialize # Initializes a cluster
scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
scheduler-k3s:report [<app>] [<flag>] # Displays a scheduler-k3s report for one or more apps
@@ -416,6 +417,56 @@ Each value in the `metadata` stanza can use the following interpolated strings:
- `DOKKU_PROCESS_TYPE`: The name of the process being scaled
- `DOKKU_APP_NAME`: The name of the app being scaled
##### HTTP Autoscaling
In addition to the built-in scalers that Keda provides, Dokku also supports Keda's HTTP Add On. This requires that the addon be properly installed and configured. For existing k3s clusters, this can be performed by the `scheduler-k3s:ensure-charts` command:
```shell
dokku scheduler-k3s:ensure-charts
```
> [!NOTE]
> Users who wish to use this functionality on a cluster not managed by Dokku will need to manually install the `keda-http-add-on` into the `keda` namespace. Please consult the `keda-http-add-on` [install documentation](https://kedacore.github.io/http-add-on/install.html) for further details.
> [!WARNING]
> If the `keda-http-add-on` chart is not installed, then this trigger will be ignored.
Once the chart is configured, an `http` trigger can be specified like so:
```json
{
"formation": {
"web": {
"autoscaling": {
"min_quantity": 1,
"max_quantity": 10,
"triggers": [
{
"type": "http",
"metadata": {
"scaledown_period_seconds": "150",
"request_rate_target_value": "50"
}
}
]
}
}
}
}
```
The following metadata properties are supported with the http autoscaler:
- `scale_by`: (default: `request_rate`) whether to scale by `concurrency` or `request_rate`.
- `scaledown_period_seconds`: (default: `300`) period to wait after the last reported active before scaling the resource back to 0.
- `request_rate_granularity_seconds`: (default: `1`) granualarity of the aggregated requests for the request rate calculation.
- `request_rate_target_value`: (default: `100`) target value for the request rate.
- `request_rate_window_seconds`: (default: `60`) aggregation window for the request rate calculation.
- `concurrency_target_value`: (default: `100`) target value for the request concurrency.
Note that due to Keda limitations, scaling is done by _either_ `concurrency` or `request_rate`.
#### Workload Autoscaling Authentication
Most Keda triggers require some form of authentication to query for data. In the Kubernetes API, they are represented by `TriggerAuthentication` and `ClusterTriggerAuthentication` resources. Dokku can manage these via the `scheduler-k3s:autoscaling-auth` commands, and includes generated resources with each helm release generated by a deploy.

2
plugins/scheduler-k3s/Makefile
View File

@@ -1,4 +1,4 @@
SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop
BUILD = commands subcommands triggers
PLUGIN_NAME = scheduler-k3s

66
plugins/scheduler-k3s/functions.go
View File

@@ -584,8 +584,18 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) {
"DEPLOYMENT_NAME": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
}
validHttpScaleMethods := map[string]bool{
"request_rate": true,
"concurrency": true,
}
hasHttpTrigger := false
httpTrigger := ProcessAutoscalingTrigger{}
triggers := []ProcessAutoscalingTrigger{}
for _, trigger := range config.Triggers {
for idx, trigger := range config.Triggers {
if trigger.Type == "" {
return ProcessAutoscaling{}, fmt.Errorf("Autoscaling trigger type is required for trigger: index %d", idx)
}
metadata := map[string]string{}
for key, value := range trigger.Metadata {
tmpl, err := template.New("").Delims("[[", "]]").Parse(value)
@@ -600,33 +610,77 @@ func getAutoscaling(input GetAutoscalingInput) (ProcessAutoscaling, error) {
metadata[key] = output.String()
}
trigger := ProcessAutoscalingTrigger{
if trigger.Name == "" {
trigger.Name = fmt.Sprintf("trigger-%s-%d", trigger.Type, idx+1)
}
autoscalingTrigger := ProcessAutoscalingTrigger{
Name: trigger.Name,
Type: trigger.Type,
Metadata: metadata,
}
if auth, ok := input.KedaValues.Authentications[trigger.Type]; ok {
trigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{
autoscalingTrigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{
Name: auth.Name,
Kind: string(auth.Kind),
}
} else if auth, ok := input.KedaValues.GlobalAuthentications[trigger.Type]; ok {
trigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{
autoscalingTrigger.AuthenticationRef = &ProcessAutoscalingTriggerAuthenticationRef{
Name: auth.Name,
Kind: string(auth.Kind),
}
}
triggers = append(triggers, trigger)
if autoscalingTrigger.Type == "http" {
if hasHttpTrigger {
return ProcessAutoscaling{}, errors.New("Only one http trigger is allowed")
}
hasHttpTrigger = true
httpTrigger = autoscalingTrigger
if _, ok := metadata["scale_by"]; !ok {
httpTrigger.Metadata["scale_by"] = "request_rate"
}
if !validHttpScaleMethods[metadata["scale_by"]] {
return ProcessAutoscaling{}, fmt.Errorf("Invalid http scale method: %s", metadata["scale_by"])
}
if _, ok := metadata["scaledown_period_seconds"]; !ok {
httpTrigger.Metadata["scaledown_period_seconds"] = "300"
}
if _, ok := metadata["request_rate_granularity_seconds"]; !ok {
httpTrigger.Metadata["request_rate_granularity_seconds"] = "1"
}
if _, ok := metadata["request_rate_target_value"]; !ok {
httpTrigger.Metadata["request_rate_target_value"] = "100"
}
if _, ok := metadata["request_rate_window_seconds"]; !ok {
httpTrigger.Metadata["request_rate_window_seconds"] = "60"
}
if _, ok := metadata["concurrency_target_value"]; !ok {
httpTrigger.Metadata["concurrency_target_value"] = "100"
}
continue
}
triggers = append(triggers, autoscalingTrigger)
}
autoscaling := ProcessAutoscaling{
CooldownPeriodSeconds: ptr.Deref(config.CooldownPeriodSeconds, 300),
Enabled: len(triggers) > 0,
Enabled: len(triggers) > 0 || hasHttpTrigger,
MaxReplicas: ptr.Deref(config.MaxQuantity, 0),
MinReplicas: ptr.Deref(config.MinQuantity, 0),
PollingIntervalSeconds: ptr.Deref(config.PollingIntervalSeconds, 30),
HttpTrigger: httpTrigger,
Triggers: triggers,
Type: "keda",
}

66
plugins/scheduler-k3s/helm.go
View File

@@ -21,6 +21,7 @@ import (
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/kube"
"helm.sh/helm/v3/pkg/release"
"helm.sh/helm/v3/pkg/repo"
"helm.sh/helm/v3/pkg/storage/driver"
)
@@ -58,9 +59,12 @@ type ChartInput struct {
}
type Release struct {
Name string
Namespace string
Version int
AppVersion string
Name string
Namespace string
Revision int
Status release.Status
Version string
}
type HelmAgent struct {
@@ -269,24 +273,61 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error {
return nil
}
func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Release, error) {
client := action.NewHistory(h.Configuration)
response, err := client.Run(releaseName)
func (h *HelmAgent) InstalledRevision(releaseName string) (Release, error) {
revisions, err := h.ListRevisions(ListRevisionsInput{
ReleaseName: releaseName,
Max: 1,
})
if err != nil {
return nil, fmt.Errorf("Error getting revisions: %w", err)
return Release{}, err
}
if len(revisions) == 0 {
return Release{}, nil
}
return revisions[0], nil
}
type ListRevisionsInput struct {
ReleaseName string
Max int
}
func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) {
client := action.NewHistory(h.Configuration)
if input.Max > 0 {
client.Max = input.Max
}
releases := []Release{}
response, err := client.Run(input.ReleaseName)
if err != nil {
if errors.Is(err, driver.ErrReleaseNotFound) {
return releases, nil
}
return nil, fmt.Errorf("Error getting revisions: %w", err)
}
for _, release := range response {
appVersion := "MISSING"
if release.Chart != nil && release.Chart.Metadata != nil {
appVersion = release.Chart.AppVersion()
}
releases = append(releases, Release{
Name: release.Name,
Namespace: release.Namespace,
Version: release.Version,
AppVersion: appVersion,
Name: release.Name,
Namespace: release.Namespace,
Revision: release.Version,
Status: release.Info.Status,
Version: release.Chart.Metadata.Version,
})
}
sort.Slice(releases, func(i, j int) bool {
return releases[i].Version < releases[j].Version
return releases[i].Revision < releases[j].Revision
})
return releases, nil
@@ -334,7 +375,8 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
client.RepoURL = input.RepoURL
}
chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, nil)
settings := cli.New()
chart, err := client.ChartPathOptions.LocateChart(input.ChartPath, settings)
if err != nil {
return fmt.Errorf("Error locating chart: %w", err)
}

8
plugins/scheduler-k3s/scheduler_k3s.go
View File

@@ -122,6 +122,14 @@ var HelmCharts = []HelmChart{
RepoURL: "https://kedacore.github.io/charts",
Version: "2.13.1",
},
{
ChartPath: "keda-add-ons-http",
CreateNamespace: true,
Namespace: "keda",
ReleaseName: "keda-add-ons-http",
RepoURL: "https://kedacore.github.io/charts",
Version: "0.8.0",
},
}
type HelmRepository struct {

1
plugins/scheduler-k3s/src/commands/commands.go
View File

@@ -22,6 +22,7 @@ Additional commands:`
scheduler-k3s:cluster-add [--insecure-allow-unknown-hosts] [--server-ip SERVER_IP] [--taint-scheduling] <ssh://user@host:port>, Adds a server node to a Dokku-managed cluster
scheduler-k3s:cluster-list [--format json|stdout], Lists all nodes in a Dokku-managed cluster
scheduler-k3s:cluster-remove [node-id], Removes client node to a Dokku-managed cluster
scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster
scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
scheduler-k3s:report [<app>] [<flag>], Displays a scheduler-k3s report for one or more apps

4
plugins/scheduler-k3s/src/subcommands/subcommands.go
View File

@@ -73,6 +73,10 @@ func main() {
args.Parse(os.Args[2:])
nodeName := args.Arg(0)
err = scheduler_k3s.CommandClusterRemove(nodeName)
case "ensure-charts":
args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError)
args.Parse(os.Args[2:])
err = scheduler_k3s.CommandEnsureCharts()
case "initialize":
args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError)
taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")

85
plugins/scheduler-k3s/subcommands.go
View File

@@ -798,6 +798,91 @@ func CommandClusterRemove(nodeName string) error {
return nil
}
// CommandEnsureCharts ensures that the required helm charts are installed
func CommandEnsureCharts() error {
ctx, cancel := context.WithCancel(context.Background())
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGQUIT,
syscall.SIGTERM)
go func() {
<-signals
cancel()
}()
clientset, err := NewKubernetesClient()
if err != nil {
return fmt.Errorf("Unable to create kubernetes client: %w", err)
}
ingressClass := getGlobalIngressClass()
namespacedHelmAgents := map[string]*HelmAgent{}
for _, chart := range HelmCharts {
_, ok := namespacedHelmAgents[chart.Namespace]
if !ok {
helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter)
if err != nil {
common.LogWarn(fmt.Sprintf("Unable to create helm agent: %s", err.Error()))
return err
}
namespacedHelmAgents[chart.Namespace] = helmAgent
}
}
common.LogInfo2Quiet("Installing helm charts")
err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool {
common.LogInfo1(fmt.Sprintf("Processing chart %s@%s", chart.ReleaseName, chart.Version))
if chart.ChartPath == "traefik" && ingressClass == "nginx" {
common.LogVerbose("Skipping chart due to ingress-class mismatch")
return false
}
if chart.ChartPath == "ingress-nginx" && ingressClass == "traefik" {
common.LogVerbose("Skipping chart due to ingress-class mismatch")
return false
}
helmAgent := namespacedHelmAgents[chart.Namespace]
latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName)
if err != nil {
common.LogWarn(fmt.Sprintf("Unable to get installed revision: %s", err))
return false
}
if latestRevision.Name == "" {
common.LogVerbose("Installing missing chart")
return true
}
if latestRevision.Version != chart.Version {
common.LogVerbose(fmt.Sprintf("Installing chart due to version mismatch: %s != %s", latestRevision.AppVersion, chart.Version))
return false
}
common.LogVerbose("Skipping chart: already installed")
return false
})
if err != nil {
return fmt.Errorf("Unable to install helm charts: %w", err)
}
for _, manifest := range KubernetesManifests {
common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version))
err = clientset.ApplyKubernetesManifest(ctx, ApplyKubernetesManifestInput{
Manifest: manifest.Path,
})
if err != nil {
return fmt.Errorf("Unable to apply kubernetes manifest: %w", err)
}
}
common.LogInfo2Quiet("Done")
return nil
}
// CommandLabelsSet set or clear a scheduler-k3s label for an app
func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error {
if resourceType == "" {

40
plugins/scheduler-k3s/template.go
View File

@@ -61,8 +61,14 @@ type GlobalImage struct {
}
type GlobalNetwork struct {
// IngressClass is the default ingress class to use
IngressClass string `yaml:"ingress_class"`
PrimaryPort int32 `yaml:"primary_port"`
// PrimaryPort is the primary port to use
PrimaryPort int32 `yaml:"primary_port"`
// PrimaryServicePort is the primary service port to use
PrimaryServicePort int32 `yaml:"primary_service_port"`
}
// GlobalKedaValues contains the global keda configuration
@@ -116,6 +122,8 @@ type ProcessAnnotations struct {
IngressAnnotations map[string]string `yaml:"ingress,omitempty"`
JobAnnotations map[string]string `yaml:"job,omitempty"`
KedaScalingObjectAnnotations map[string]string `yaml:"keda_scaled_object,omitempty"`
KedaHTTPScaledObjectAnnotations map[string]string `yaml:"keda_http_scaled_object,omitempty"`
KedaInterceptorProxyAnnotations map[string]string `yaml:"keda_interceptor_proxy,omitempty"`
KedaSecretAnnotations map[string]string `yaml:"keda_secret,omitempty"`
KedaTriggerAuthenticationAnnotations map[string]string `yaml:"keda_trigger_authentication,omitempty"`
PodAnnotations map[string]string `yaml:"pod,omitempty"`
@@ -143,6 +151,9 @@ type ProcessAutoscaling struct {
// PollingIntervalSeconds is the number of seconds between polling for new metrics
PollingIntervalSeconds int `yaml:"polling_interval_seconds,omitempty"`
// HttpTrigger is the http trigger config to use for autoscaling
HttpTrigger ProcessAutoscalingTrigger `yaml:"http_trigger,omitempty"`
// Triggers is a list of triggers to use for autoscaling
Triggers []ProcessAutoscalingTrigger `yaml:"triggers,omitempty"`
@@ -224,17 +235,22 @@ const (
)
type ProcessLabels struct {
CertificateLabels map[string]string `yaml:"certificate,omitempty"`
CronJobLabels map[string]string `yaml:"cronjob,omitempty"`
DeploymentLabels map[string]string `yaml:"deployment,omitempty"`
IngressLabels map[string]string `yaml:"ingress,omitempty"`
JobLabels map[string]string `yaml:"job,omitempty"`
PodLabels map[string]string `yaml:"pod,omitempty"`
SecretLabels map[string]string `yaml:"secret,omitempty"`
ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"`
ServiceLabels map[string]string `yaml:"service,omitempty"`
TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"`
TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"`
CertificateLabels map[string]string `yaml:"certificate,omitempty"`
CronJobLabels map[string]string `yaml:"cronjob,omitempty"`
DeploymentLabels map[string]string `yaml:"deployment,omitempty"`
IngressLabels map[string]string `yaml:"ingress,omitempty"`
JobLabels map[string]string `yaml:"job,omitempty"`
KedaScalingObjectLabels map[string]string `yaml:"keda_scaled_object,omitempty"`
KedaHTTPScaledObjectLabels map[string]string `yaml:"keda_http_scaled_object,omitempty"`
KedaInterceptorProxyLabels map[string]string `yaml:"keda_interceptor_proxy,omitempty"`
KedaSecretLabels map[string]string `yaml:"keda_secret,omitempty"`
KedaTriggerAuthenticationLabels map[string]string `yaml:"keda_trigger_authentication,omitempty"`
PodLabels map[string]string `yaml:"pod,omitempty"`
SecretLabels map[string]string `yaml:"secret,omitempty"`
ServiceAccountLabels map[string]string `yaml:"serviceaccount,omitempty"`
ServiceLabels map[string]string `yaml:"service,omitempty"`
TraefikIngressRouteLabels map[string]string `yaml:"traefik_ingressroute,omitempty"`
TraefikMiddlewareLabels map[string]string `yaml:"traefik_middleware,omitempty"`
}
type ProcessWeb struct {

14
plugins/scheduler-k3s/templates/chart/ingress.yaml
View File

@@ -1,5 +1,11 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := index .Values.processes "PROCESS_NAME" }}
{{- $routeToKeda := false }}
{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }}
{{- $routeToKeda = true }}
{{- end }}
{{- end }}
{{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }}
{{- range $pdx, $domain := $config.web.domains }}
---
@@ -36,11 +42,19 @@ spec:
paths:
{{- if $config.web.port_maps }}
- backend:
{{- if $routeToKeda}}
service:
name: kip-{{ $.Values.global.app_name}}-{{ $processName }}
port:
number: 8080
pathType: Prefix
{{- else }}
service:
name: {{ $.Values.global.app_name }}-{{ $processName }}
port:
name: {{ include "primary.port" $config.web.port_maps }}
pathType: ImplementationSpecific
{{- end }}
path: /
{{- end }}
{{- end }}

53
plugins/scheduler-k3s/templates/chart/keda-http-scaled-object.yaml Normal file
View File

@@ -0,0 +1,53 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := index .Values.processes "PROCESS_NAME" }}
{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
{{- if and (.Capabilities.APIVersions.Has "http.keda.sh/v1alpha1") (and $config.autoscaling.http_trigger (eq $config.autoscaling.http_trigger.type "http")) }}
{{- $mappings := dict }}
{{- range $pdx, $port_map := $config.web.port_maps }}
{{- $mappings := set $mappings $port_map.name "true" }}
{{- end }}
---
kind: HTTPScaledObject
apiVersion: http.keda.sh/v1alpha1
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
dokku.com/managed: "true"
httpscaledobject.keda.sh/skip-scaledobject-creation: "true"
{{ include "print.annotations" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }}
{{ include "print.annotations" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }}
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
{{ include "print.labels" (dict "config" $.Values.global "key" "keda_http_scaled_object") | indent 4 }}
{{ include "print.labels" (dict "config" $config "key" "keda_http_scaled_object") | indent 4 }}
name: {{ $.Values.global.app_name}}-{{ $processName }}
namespace: {{ $.Values.global.namespace }}
spec:
hosts:
{{- range $pdx, $domain := $config.web.domains }}
- {{ $domain.name | quote }}
{{- end }}
scaleTargetRef:
name: {{ $.Values.global.app_name }}-{{ $processName }}
kind: Deployment
apiVersion: apps/v1
service: {{ $.Values.global.app_name }}-{{ $processName }}
port: {{ $.Values.global.network.primary_service_port }}
replicas:
min: {{ $config.autoscaling.min_replicas }}
max: {{ $config.autoscaling.max_replicas }}
scaledownPeriod: {{ get $config.autoscaling.http_trigger.metadata "scaledown_period_seconds" }}
scalingMetric:
{{- if eq (get $config.autoscaling.http_trigger.metadata "scale_by") "request_rate" }}
requestRate:
granularity: {{ get $config.autoscaling.http_trigger.metadata "request_rate_granularity_seconds" }}s
targetValue: {{ get $config.autoscaling.http_trigger.metadata "request_rate_target_value" }}
window: {{ get $config.autoscaling.http_trigger.metadata "request_rate_window_seconds" }}s
{{- else }}
concurrency:
targetValue: {{ get $config.autoscaling.http_trigger.metadata "concurrency_target_value" }}
{{- end }}
{{- end }}
{{- end }}

26
plugins/scheduler-k3s/templates/chart/keda-interceptor-proxy-service.yaml Normal file
View File

@@ -0,0 +1,26 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := index .Values.processes "PROCESS_NAME" }}
{{- if and (eq $processName "web") (and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda"))) }}
{{- if .Capabilities.APIVersions.Has "http.keda.sh/v1alpha1" -}}
---
kind: Service
apiVersion: v1
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
dokku.com/managed: "true"
{{ include "print.annotations" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }}
{{ include "print.annotations" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }}
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
{{ include "print.labels" (dict "config" $.Values.global "key" "keda_interceptor_proxy_service") | indent 4 }}
{{ include "print.labels" (dict "config" $config "key" "keda_interceptor_proxy_service") | indent 4 }}
name: kip-{{ $.Values.global.app_name}}-{{ $processName }}
namespace: {{ $.Values.global.namespace }}
spec:
type: ExternalName
externalName: keda-add-ons-http-interceptor-proxy.keda.svc.cluster.local
{{- end }}
{{- end }}

5
plugins/scheduler-k3s/templates/chart/keda-scaled-object.yaml
View File

@@ -2,6 +2,7 @@
{{- $config := index .Values.processes "PROCESS_NAME" }}
{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }}
{{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}}
---
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
@@ -31,9 +32,13 @@ spec:
fallback:
failureThreshold: 3
replicas: {{ $config.replicas }}
{{- if $config.autoscaling.triggers }}
triggers:
{{- with $config.autoscaling.triggers }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- else }}
triggers: []
{{- end }}
{{- end }}
{{- end }}

31
plugins/scheduler-k3s/templates/helm-config/keda-add-ons-http.yaml Normal file
View File

@@ -0,0 +1,31 @@
---
operator:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
scaler:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
interceptor:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule

9
plugins/scheduler-k3s/triggers.go
View File

@@ -250,8 +250,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
}
primaryPort := int32(5000)
primaryServicePort := int32(80)
for _, portMap := range portMaps {
primaryPort = portMap.ContainerPort
primaryServicePort = portMap.HostPort
if primaryPort != 0 {
break
}
@@ -350,8 +352,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
Labels: globalLabels,
Namespace: namespace,
Network: GlobalNetwork{
IngressClass: getGlobalIngressClass(),
PrimaryPort: primaryPort,
IngressClass: getGlobalIngressClass(),
PrimaryPort: primaryPort,
PrimaryServicePort: primaryServicePort,
},
Secrets: map[string]string{},
},
@@ -504,7 +507,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
templateFiles := []string{"deployment", "keda-scaled-object"}
if processType == "web" {
templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware")
templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
}
for _, templateName := range templateFiles {
b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))

2
tests/unit/git_3.bats
View File

@@ -5,7 +5,7 @@ load test_helper
SMOKE_TEST_APP_1_0_0_SHA=9cf71bba639c4f1671dfd42685338b762d3354f2
SMOKE_TEST_APP_2_0_0_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9
SMOKE_TEST_APP_ANOTHER_BRANCH_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9
SMOKE_TEST_APP_MASTER_SHA=af1b02052199b8ca8115f80ff8676a7c7744a45f
SMOKE_TEST_APP_MASTER_SHA=96f6ba2db912c93707080a358016d1febb7c560a
SMOKE_TEST_APP_COMMIT_SHA=5c8a5e42bbd7fae98bd657fb17f41c6019b303f9
setup() {