github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: add initial support for injecting keda addons

Browse Source
This commit is contained in:
Jose Diaz-Gonzalez
2024-02-27 14:26:16 -05:00
parent 3242073e84
commit b7ff696507
11 changed files with 223 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
plugins/app-json/appjson.go
View File

@@ -3,10 +3,12 @@ package appjson
import (
"encoding/json"
"fmt"
"math"
"os"
"strings"
"github.com/dokku/dokku/plugins/common"
"k8s.io/utils/ptr"
)
var (
@@ -59,6 +61,9 @@ type CronCommand struct {
// Formation is a struct that represents the scale for a process from an app.json file
type Formation struct {
// Autoscaling is whether or not to enable autoscaling
Autoscaling *FormationAutoscaling `json:"autoscaling"`
// Quantity is the number of processes to run
Quantity *int `json:"quantity"`
@@ -66,6 +71,36 @@ type Formation struct {
MaxParallel *int `json:"max_parallel"`
}
// FormationAutoscaling is a struct that represents the autoscaling configuration for a process from an app.json file
type FormationAutoscaling struct {
// CoolDownSeconds is the number of seconds to wait before scaling again
CooldownPeriodSeconds *int `json:"cooldown_period_seconds,omitempty"`
// MaxQuantity is the maximum number of processes to run
MaxQuantity *int `json:"max_quantity,omitempty"`
// MinQuantity is the minimum number of processes to run
MinQuantity *int `json:"min_quantity,omitempty"`
// PollingIntervalSeconds is the number of seconds to wait between autoscaling checks
PollingIntervalSeconds *int `json:"polling_interval_seconds,omitempty"`
// Triggers is a list of triggers to use for autoscaling
Triggers []FormationAutoscalingTrigger `json:"triggers,omitempty"`
}
// FormationAutoscalingTrigger is a struct that represents a single autoscaling trigger from an app.json file
type FormationAutoscalingTrigger struct {
// Name is the name of the trigger
Name string `json:"name,omitempty"`
// Type is the type of the trigger
Type string `json:"type,omitempty"`
// Metadata is a map of metadata to use for the trigger
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
// Healthcheck is a struct that represents a single healthcheck from an app.json file
type Healthcheck struct {
// Attempts is the number of attempts to make before considering a healthcheck failed
@@ -181,3 +216,45 @@ func GetAppJSON(appName string) (AppJSON, error) {
return appJSON, nil
}
func GetAutoscalingConfig(appName string, processType string, replicas int) (FormationAutoscaling, bool, error) {
appJSON, err := GetAppJSON(appName)
if err != nil {
return FormationAutoscaling{}, false, err
}
common.LogWarn(fmt.Sprintf("appJSON: %v", appJSON))
formation, ok := appJSON.Formation[processType]
if !ok {
return FormationAutoscaling{}, false, nil
}
if formation.Autoscaling == nil {
return FormationAutoscaling{}, false, nil
}
autoscaling := *formation.Autoscaling
if autoscaling.CooldownPeriodSeconds == nil {
autoscaling.CooldownPeriodSeconds = ptr.To(300)
}
if autoscaling.MinQuantity == nil {
autoscaling.MinQuantity = ptr.To(replicas)
}
if autoscaling.MaxQuantity == nil {
defaultValue := math.Max(float64(replicas), float64(*autoscaling.MinQuantity))
autoscaling.MaxQuantity = ptr.To(int(defaultValue))
}
if autoscaling.PollingIntervalSeconds == nil {
autoscaling.PollingIntervalSeconds = ptr.To(30)
}
if len(autoscaling.Triggers) == 0 {
return FormationAutoscaling{}, false, nil
}
return autoscaling, true, nil
}

1
plugins/app-json/go.mod
View File

@@ -7,6 +7,7 @@ require (
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/spf13/pflag v1.0.5
golang.org/x/sync v0.6.0
k8s.io/utils v0.0.0-20240102154912-e7106e64919e
)
require (

5
plugins/app-json/go.sum
View File

@@ -4,8 +4,9 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -95,3 +96,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=

1
plugins/cron/go.mod
View File

@@ -30,6 +30,7 @@ require (
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.18.0 // indirect
k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
)
replace github.com/dokku/dokku/plugins/app-json => ../app-json

5
plugins/cron/go.sum
View File

@@ -4,8 +4,9 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -107,5 +108,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
mvdan.cc/sh/v3 v3.8.0 h1:ZxuJipLZwr/HLbASonmXtcvvC9HXY9d2lXZHnKGjFc8=
mvdan.cc/sh/v3 v3.8.0/go.mod h1:w04623xkgBVo7/IUK89E0g8hBykgEpN0vgOj3RJr6MY=

1
plugins/scheduler-docker-local/go.mod
View File

@@ -29,6 +29,7 @@ require (
github.com/ryanuber/columnize v2.1.2+incompatible // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/sys v0.18.0 // indirect
k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
mvdan.cc/sh/v3 v3.8.0 // indirect
)

5
plugins/scheduler-docker-local/go.sum
View File

@@ -4,8 +4,9 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -105,5 +106,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
mvdan.cc/sh/v3 v3.8.0 h1:ZxuJipLZwr/HLbASonmXtcvvC9HXY9d2lXZHnKGjFc8=
mvdan.cc/sh/v3 v3.8.0/go.mod h1:w04623xkgBVo7/IUK89E0g8hBykgEpN0vgOj3RJr6MY=

42
plugins/scheduler-k3s/functions.go
View File

@@ -27,6 +27,7 @@ import (
"k8s.io/client-go/tools/remotecommand"
"k8s.io/kubectl/pkg/util/term"
"k8s.io/kubernetes/pkg/client/conditions"
"k8s.io/utils/ptr"
"mvdan.cc/sh/v3/shell"
)
@@ -395,6 +396,12 @@ func getAnnotations(appName string, processType string) (ProcessAnnotations, err
}
annotations.JobAnnotations = jobAnnotations
kedaScalingObjectAnnotations, err := getAnnotation(appName, processType, "keda_scaled_object")
if err != nil {
return annotations, err
}
annotations.KedaScalingObjectAnnotations = kedaScalingObjectAnnotations
podAnnotations, err := getAnnotation(appName, processType, "pod")
if err != nil {
return annotations, err
@@ -434,6 +441,41 @@ func getAnnotations(appName string, processType string) (ProcessAnnotations, err
return annotations, nil
}
// getAutoscaling retrieves autoscaling config for a given app and process type
func getAutoscaling(appName string, processName string, replicas int) (ProcessAutoscaling, error) {
config, ok, err := appjson.GetAutoscalingConfig(appName, processName, replicas)
if err != nil {
common.LogWarn(fmt.Sprintf("Error getting autoscaling config for %s: %v", appName, err))
return ProcessAutoscaling{}, err
}
if !ok {
common.LogWarn(fmt.Sprintf("No autoscaling config found for %s", appName))
return ProcessAutoscaling{}, nil
}
triggers := []ProcessAutoscalingTrigger{}
for _, trigger := range config.Triggers {
triggers = append(triggers, ProcessAutoscalingTrigger{
Name: trigger.Name,
Type: trigger.Type,
Metadata: trigger.Metadata,
})
}
autoscaling := ProcessAutoscaling{
CooldownPeriodSeconds: ptr.Deref(config.CooldownPeriodSeconds, 300),
Enabled: len(triggers) > 0,
MaxReplicas: ptr.Deref(config.MaxQuantity, 0),
MinReplicas: ptr.Deref(config.MinQuantity, 0),
PollingIntervalSeconds: ptr.Deref(config.PollingIntervalSeconds, 30),
Triggers: triggers,
Type: "keda",
}
return autoscaling, nil
}
// getGlobalAnnotations retrieves global annotations for a given app
func getGlobalAnnotations(appName string) (ProcessAnnotations, error) {
return getAnnotations(appName, GlobalProcessType)

38
plugins/scheduler-k3s/template.go
View File

@@ -59,6 +59,7 @@ type GlobalNetwork struct {
type ProcessValues struct {
Annotations ProcessAnnotations `yaml:"annotations,omitempty"`
Args []string `yaml:"args,omitempty"`
Autoscaling ProcessAutoscaling `yaml:"autoscaling,omitempty"`
Cron ProcessCron `yaml:"cron,omitempty"`
Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"`
Labels ProcessLabels `yaml:"labels,omitempty"`
@@ -74,6 +75,7 @@ type ProcessAnnotations struct {
DeploymentAnnotations map[string]string `yaml:"deployment,omitempty"`
IngressAnnotations map[string]string `yaml:"ingress,omitempty"`
JobAnnotations map[string]string `yaml:"job,omitempty"`
KedaScalingObjectAnnotations map[string]string `yaml:"keda_scaled_object,omitempty"`
PodAnnotations map[string]string `yaml:"pod,omitempty"`
SecretAnnotations map[string]string `yaml:"secret,omitempty"`
ServiceAccountAnnotations map[string]string `yaml:"serviceaccount,omitempty"`
@@ -82,6 +84,42 @@ type ProcessAnnotations struct {
TraefikMiddlewareAnnotations map[string]string `yaml:"traefik_middleware,omitempty"`
}
// ProcessAutoscaling contains the autoscaling configuration for a process
type ProcessAutoscaling struct {
// CooldownPeriodSeconds is the number of seconds after a scaling event before another can be triggered
CooldownPeriodSeconds int `yaml:"cooldown_period_seconds,omitempty"`
// Enabled is a flag to enable autoscaling
Enabled bool `yaml:"enabled"`
// MaxReplicas is the maximum number of replicas to scale to
MaxReplicas int `yaml:"max_replicas,omitempty"`
// MinReplicas is the minimum number of replicas to scale to
MinReplicas int `yaml:"min_replicas,omitempty"`
// PollingIntervalSeconds is the number of seconds between polling for new metrics
PollingIntervalSeconds int `yaml:"polling_interval_seconds,omitempty"`
// Triggers is a list of triggers to use for autoscaling
Triggers []ProcessAutoscalingTrigger `yaml:"triggers,omitempty"`
// Type is the type of autoscaling to use
Type string `yaml:"type"`
}
// ProcessAutoscalingTrigger is a trigger to use for autoscaling
type ProcessAutoscalingTrigger struct {
// Name is the name of the trigger
Name string `yaml:"name"`
// Type is the type of trigger to use
Type string `yaml:"type"`
// Metadata is a map of key-value pairs that can be used to store arbitrary trigger data
Metadata map[string]interface{} `yaml:"metadata,omitempty"`
}
type ProcessHealthchecks struct {
Liveness ProcessHealthcheck `yaml:"liveness,omitempty"`
Readiness ProcessHealthcheck `yaml:"readiness,omitempty"`

43
plugins/scheduler-k3s/templates/chart/scaled-object.yaml Normal file
View File

@@ -0,0 +1,43 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := index .Values.processes "PROCESS_NAME" }}
{{- if and $config.autoscaling (and $config.autoscaling.enabled (eq $config.autoscaling.type "keda")) }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
dokku.com/builder-type: {{ $.Values.global.image.type }}
dokku.com/managed: "true"
kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
{{- if and $.Values.global.annotations $.Values.global.annotations.keda_scaled_object }}
{{- range $k, $v := $.Values.global.annotations.keda_scaled_object }}
{{ $k }}: {{ $v | quote }}
{{- end }}
{{- end }}
{{- if and $config.annotations $config.annotations.keda_scaled_object }}
{{- range $k, $v := $config.annotations.keda_scaled_object }}
{{ $k }}: {{ $v | quote }}
{{- end }}
{{- end }}
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}-scaled-object
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
name: {{ $.Values.global.app_name}}-{{ $processName }}
namespace: {{ $.Values.global.namespace }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ $.Values.global.app_name }}-{{ $processName }}
envSourceContainerName: {{ $.Values.global.app_name }}-{{ $processName }}
cooldownPeriod: {{ $config.autoscaling.cooldown_period_seconds }}
pollingInterval: {{ $config.autoscaling.polling_interval_seconds }}
minReplicaCount: {{ $config.autoscaling.min_replicas }}
maxReplicaCount: {{ $config.autoscaling.max_replicas }}
fallback:
failureThreshold: 3
replicas: {{ $config.replicas }}
triggers:
{{- toYaml $config.autoscaling.triggers | nindent 4 }}
{{- end }}

9
plugins/scheduler-k3s/triggers.go
View File

@@ -327,8 +327,14 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
return fmt.Errorf("Error getting process labels: %w", err)
}
autoscaling, err := getAutoscaling(appName, processType, int(processCount))
if err != nil {
return fmt.Errorf("Error getting autoscaling: %w", err)
}
processValues := ProcessValues{
Annotations: annotations,
Autoscaling: autoscaling,
Args: args,
Healthchecks: processHealthchecks,
Labels: labels,
@@ -336,6 +342,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
Replicas: int32(processCount),
Resources: processResources,
}
if processType == "web" {
processValues.Web = ProcessWeb{
Domains: domains,
@@ -392,7 +399,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
values.Processes[processType] = processValues
templateFiles := []string{"deployment"}
templateFiles := []string{"deployment", "scaled-object"}
if processType == "web" {
templateFiles = append(templateFiles, "service", "certificate", "ingress", "ingress-route", "https-redirect-middleware")
}