refactor: move templating into pseudo-generated helm chart

The previous mechanism of constructing the helm chart was pretty obnoxious due to needing to have some things in the object and some in template files that were appended afterwards. This change simplifies that by using template files per type with minimal replacements.
This commit is contained in:
Jose Diaz-Gonzalez committed 2024-01-23 03:00:27 -05:00
1 parent 75539273e3
commit 2336063ca2
15 files changed
+752 -970

No files matched your search

+148 -1
View File
@@ -10,10 +10,12 @@ import (
"strings" "strings"
"time" "time"
appjson "github.com/dokku/dokku/plugins/app-json"
"github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/common"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -355,6 +357,151 @@ func getGlobalGlobalToken() string {
return common.PropertyGet("scheduler-k3s", "--global", "token") return common.PropertyGet("scheduler-k3s", "--global", "token")
} }
func getProcessHealtchecks(healthchecks []appjson.Healthcheck, primaryPort int32) ProcessHealthchecks {
if len(healthchecks) == 0 {
return ProcessHealthchecks{}
}
livenessChecks := []ProcessHealthcheck{}
readinessChecks := []ProcessHealthcheck{}
startupChecks := []ProcessHealthcheck{}
uptimeSeconds := []int32{}
for _, healthcheck := range healthchecks {
probe := ProcessHealthcheck{
InitialDelaySeconds: healthcheck.InitialDelay,
PeriodSeconds: healthcheck.Wait,
TimeoutSeconds: healthcheck.Timeout,
FailureThreshold: healthcheck.Attempts,
SuccessThreshold: int32(1),
}
if len(healthcheck.Command) > 0 {
probe.Exec = &ExecHealthcheck{
Command: healthcheck.Command,
}
} else if healthcheck.Listening {
probe.TCPSocket = &TCPHealthcheck{
Port: primaryPort,
}
for _, header := range healthcheck.HTTPHeaders {
if header.Name == "Host" {
probe.TCPSocket.Host = header.Value
}
}
} else if healthcheck.Path != "" {
probe.HTTPGet = &HTTPHealthcheck{
Path: healthcheck.Path,
Port: primaryPort,
HTTPHeaders: []HTTPHeader{},
}
if healthcheck.Scheme != "" {
probe.HTTPGet.Scheme = URIScheme(strings.ToUpper(healthcheck.Scheme))
}
for _, header := range healthcheck.HTTPHeaders {
probe.HTTPGet.HTTPHeaders = append(probe.HTTPGet.HTTPHeaders, HTTPHeader{
Name: header.Name,
Value: header.Value,
})
}
} else if healthcheck.Uptime > 0 {
uptimeSeconds = append(uptimeSeconds, healthcheck.Uptime)
}
if healthcheck.Type == appjson.HealthcheckType_Liveness {
livenessChecks = append(livenessChecks, probe)
} else if healthcheck.Type == appjson.HealthcheckType_Readiness {
readinessChecks = append(readinessChecks, probe)
} else if healthcheck.Type == appjson.HealthcheckType_Startup {
startupChecks = append(startupChecks, probe)
}
}
if len(livenessChecks) > 1 {
common.LogWarn("Multiple liveness checks are not supported, only the first one will be used")
}
if len(readinessChecks) > 1 {
common.LogWarn("Multiple readiness checks are not supported, only the first one will be used")
}
if len(startupChecks) > 1 {
common.LogWarn("Multiple startup checks are not supported, only the first one will be used")
}
if len(uptimeSeconds) > 1 {
common.LogWarn("Multiple uptime checks are not supported, only the first one will be used")
}
processHealthchecks := ProcessHealthchecks{}
if len(livenessChecks) > 0 {
processHealthchecks.Liveness = livenessChecks[0]
}
if len(readinessChecks) > 0 {
processHealthchecks.Readiness = readinessChecks[0]
}
if len(startupChecks) > 0 {
processHealthchecks.Startup = startupChecks[0]
}
if len(uptimeSeconds) > 0 {
processHealthchecks.MinReadySeconds = uptimeSeconds[0]
}
return processHealthchecks
}
func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) {
processResources := ProcessResourcesMap{
Limits: ProcessResources{
CPU: "1000m",
Memory: "512Mi",
},
Requests: ProcessResources{
CPU: "1000m",
Memory: "512Mi",
},
}
cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "cpu"}...)
if err != nil && cpuLimit != "" && cpuLimit != "0" {
_, err := resource.ParseQuantity(cpuLimit)
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err)
}
processResources.Limits.CPU = cpuLimit
}
nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...)
if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
_, err := resource.ParseQuantity(nvidiaGpuLimit)
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
}
processResources.Limits.NvidiaGPU = nvidiaGpuLimit
}
memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "memory"}...)
if err != nil && memoryLimit != "" && memoryLimit != "0" {
_, err := resource.ParseQuantity(memoryLimit)
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err)
}
processResources.Limits.Memory = memoryLimit
}
cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "cpu"}...)
if err != nil && cpuRequest != "" && cpuRequest != "0" {
_, err := resource.ParseQuantity(cpuRequest)
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err)
}
processResources.Requests.CPU = cpuRequest
}
memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "memory"}...)
if err != nil && memoryRequest != "" && memoryRequest != "0" {
_, err := resource.ParseQuantity(memoryRequest)
if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err)
}
processResources.Requests.Memory = memoryRequest
}
return processResources, nil
}
func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
command := extractStartCommand(input) command := extractStartCommand(input)
fields, err := shell.Fields(command, func(name string) string { fields, err := shell.Fields(command, func(name string) string {
@@ -407,7 +554,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error {
} }
} }
contents, err := templates.ReadFile(fmt.Sprintf("templates/%s.yaml", chart.ReleaseName)) contents, err := templates.ReadFile(fmt.Sprintf("templates/helm-config/%s.yaml", chart.ReleaseName))
if err != nil && !errors.Is(err, os.ErrNotExist) { if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err) return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err)
} }
+1 -1
View File
@@ -258,7 +258,7 @@ func CommandInitialize(taintScheduling bool) error {
} }
common.LogInfo2Quiet("Updating traefik config") common.LogInfo2Quiet("Updating traefik config")
contents, err := templates.ReadFile("templates/traefik-config.yaml") contents, err := templates.ReadFile("templates/helm-config/traefik-config.yaml")
if err != nil { if err != nil {
return fmt.Errorf("Unable to read traefik config template: %w", err) return fmt.Errorf("Unable to read traefik config template: %w", err)
} }
+116 -758
View File
@@ -1,30 +1,20 @@
package scheduler_k3s package scheduler_k3s
import ( import (
"bytes"
"crypto/rand" "crypto/rand"
"fmt" "fmt"
"os" "os"
"path/filepath"
"strings" "strings"
acmev1 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1" acmev1 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
appjson "github.com/dokku/dokku/plugins/app-json"
"github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/common"
traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
orderedmap "github.com/wk8/go-ordered-map/v2"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1" batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/cli-runtime/pkg/printers"
"k8s.io/utils/ptr" "k8s.io/utils/ptr"
) )
@@ -36,189 +26,141 @@ type Chart struct {
} }
type Values struct { type Values struct {
DeploymentID string `yaml:"deploment_id"` Global GlobalValues `yaml:"global"`
Secrets map[string]string `yaml:"secrets,omitempty"` Processes map[string]ProcessValues `yaml:"processes"`
Processes map[string]ProcessValues `yaml:"processes"` }
type GlobalValues struct {
AppName string `yaml:"app_name"`
DeploymentID string `yaml:"deploment_id"`
Image GlobalImage `yaml:"image"`
Namespace string `yaml:"namespace"`
PrimaryPort int32 `yaml:"primary_port"`
Secrets map[string]string `yaml:"secrets,omitempty"`
}
type GlobalImage struct {
ImagePullSecrets string `yaml:"image_pull_secrets"`
Name string `yaml:"name"`
Type string `yaml:"type"`
WorkingDir string `yaml:"working_dir"`
} }
type ProcessValues struct { type ProcessValues struct {
Domains []string `yaml:"domains,omitempty"` Args []string `yaml:"args,omitempty"`
Replicas int32 `yaml:"replicas"` Cron ProcessCron `yaml:"cron,omitempty"`
TLS bool `yaml:"tls"` Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"`
ProcessType ProcessType `yaml:"process_type"`
Replicas int32 `yaml:"replicas"`
Resources ProcessResourcesMap `yaml:"resources,omitempty"`
Web ProcessWeb `yaml:"web,omitempty"`
} }
type CreateIngressRoutesInput struct { type ProcessHealthchecks struct {
AppName string Liveness ProcessHealthcheck `yaml:"liveness,omitempty"`
ChartDir string Readiness ProcessHealthcheck `yaml:"readiness,omitempty"`
Deployment appsv1.Deployment Startup ProcessHealthcheck `yaml:"startup,omitempty"`
Namespace string MinReadySeconds int32 `yaml:"min_ready_seconds,omitempty"`
ProcessType string
PortMaps []PortMap
Service v1.Service
} }
func createIngressRoutesFiles(input CreateIngressRoutesInput) error { type ProcessHealthcheck struct {
for _, portMap := range input.PortMaps { Exec *ExecHealthcheck `yaml:"exec,omitempty"`
ingressRoute := templateKubernetesIngressRoute(IngressRoute{ HTTPGet *HTTPHealthcheck `yaml:"httpGet,omitempty"`
AppName: input.AppName, TCPSocket *TCPHealthcheck `yaml:"tcpSocket,omitempty"`
Namespace: input.Namespace,
PortMap: portMap,
ProcessType: input.ProcessType,
ServiceName: input.Service.Name,
})
ingressRouteFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/ingress-route-%s.yaml", portMap.String())) InitialDelaySeconds int32 `yaml:"initialDelaySeconds,omitempty"`
err := writeResourceToFile(WriteResourceInput{ TimeoutSeconds int32 `yaml:"timeoutSeconds,omitempty"`
Object: &ingressRoute, PeriodSeconds int32 `yaml:"periodSeconds,omitempty"`
Path: ingressRouteFile, SuccessThreshold int32 `yaml:"successThreshold,omitempty"`
}) FailureThreshold int32 `yaml:"failureThreshold,omitempty"`
if err != nil { TerminationGracePeriodSeconds *int64 `yaml:"terminationGracePeriodSeconds,omitempty"`
return fmt.Errorf("Error printing ingress route: %w", err)
}
b, err := os.ReadFile(ingressRouteFile)
if err != nil {
return fmt.Errorf("Error reading ingress route file: %w", err)
}
append, err := templates.ReadFile("templates/ingress-routes-append.yaml")
if err != nil {
return fmt.Errorf("Error reading ingress route append file: %w", err)
}
contents := strings.Join([]string{strings.TrimSpace(string(b)), string(append)}, "")
contents = strings.ReplaceAll(contents, " routes: null", "")
contents = strings.Join([]string{
"{{- if .Values.processes.PROCESS_TYPE.domains }}",
contents,
"{{- end }}",
}, "\n")
replacements := orderedmap.New[string, string]()
replacements.Set("PROCESS_TYPE", input.ProcessType)
replacements.Set("APP_NAME", input.AppName)
replacements.Set("NAMESPACE", input.Namespace)
replacements.Set("PORT_MAPPING", portMap.String())
replacements.Set("PORT_SCHEME", portMap.Scheme)
for pair := replacements.Oldest(); pair != nil; pair = pair.Next() {
contents = strings.ReplaceAll(contents, pair.Key, pair.Value)
}
err = os.WriteFile(ingressRouteFile, []byte(contents), os.FileMode(0644))
if err != nil {
return fmt.Errorf("Error writing ingress route file: %w", err)
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(ingressRouteFile)
}
}
return nil
} }
type CreateCertificateFileInput struct { type ExecHealthcheck struct {
Certificate Certificate Command []string `yaml:"command,omitempty"`
ChartDir string
IssuerName string
ProcessType string
} }
type Certificate struct { type HTTPHealthcheck struct {
AppName string Path string `yaml:"path,omitempty"`
Name string Port int32 `yaml:"port,omitempty"`
Namespace string Host string `yaml:"host,omitempty"`
TLS bool Scheme URIScheme `yaml:"scheme,omitempty"`
HTTPHeaders []HTTPHeader `yaml:"httpHeaders,omitempty"`
} }
func createCertificateFile(input CreateCertificateFileInput) error { type TCPHealthcheck struct {
certificate, err := templateKubernetesCertificate(input.Certificate) Port int32 `yaml:"port,omitempty"`
if err != nil { Host string `yaml:"host,omitempty"`
return fmt.Errorf("Error templating certificate: %w", err)
}
certificateFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/certificate-%s.yaml", input.ProcessType))
err = writeResourceToFile(WriteResourceInput{
Object: &certificate,
Path: certificateFile,
})
if err != nil {
return fmt.Errorf("Error printing ingress route: %w", err)
}
b, err := os.ReadFile(certificateFile)
if err != nil {
return fmt.Errorf("Error reading ingress route file: %w", err)
}
append, err := templates.ReadFile("templates/certificate-append.yaml")
if err != nil {
return fmt.Errorf("Error reading ingress route append file: %w", err)
}
contents := string(bytes.Join([][]byte{b, append}, []byte("")))
contents = strings.Join([]string{
"{{- if and .Values.processes.PROCESS_TYPE.tls .Values.processes.PROCESS_TYPE.domains }}",
contents,
"{{- end }}",
}, "\n")
replacements := orderedmap.New[string, string]()
replacements.Set("PROCESS_TYPE", input.ProcessType)
replacements.Set("ISSUER_NAME", input.IssuerName)
for pair := replacements.Oldest(); pair != nil; pair = pair.Next() {
contents = strings.ReplaceAll(contents, pair.Key, pair.Value)
}
err = os.WriteFile(certificateFile, []byte(contents), os.FileMode(0644))
if err != nil {
return fmt.Errorf("Error writing ingress route file: %w", err)
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(certificateFile)
}
return nil
} }
func templateKubernetesCertificate(input Certificate) (certmanagerv1.Certificate, error) { type HTTPHeader struct {
if input.Name == "" { Name string `yaml:"name"`
return certmanagerv1.Certificate{}, fmt.Errorf("Name cannot be empty") Value string `yaml:"value"`
} }
if input.Namespace == "" {
return certmanagerv1.Certificate{}, fmt.Errorf("Namespace cannot be empty")
}
labels := map[string]string{ type URIScheme string
"app.kubernetes.io/name": input.Name,
"app.kubernetes.io/part-of": input.AppName,
}
annotations := map[string]string{
"dokku.com/managed": "true",
}
certificate := certmanagerv1.Certificate{ const (
ObjectMeta: metav1.ObjectMeta{ URISchemeHTTP URIScheme = "HTTP"
Name: input.Name, URISchemeHTTPS URIScheme = "HTTPS"
Namespace: input.Namespace, )
Labels: labels,
Annotations: annotations,
},
Spec: certmanagerv1.CertificateSpec{
SecretName: "tls-" + input.Name,
SecretTemplate: &certmanagerv1.CertificateSecretTemplate{
Annotations: annotations,
Labels: labels,
},
IssuerRef: certmanagermetav1.ObjectReference{
Name: "ISSUER_NAME",
Kind: "ClusterIssuer",
},
},
}
return certificate, nil type ProcessWeb struct {
Domains []string `yaml:"domains,omitempty"`
PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"`
TLS bool `yaml:"tls"`
}
type ProcessResourcesMap struct {
Limits ProcessResources `yaml:"limits,omitempty"`
Requests ProcessResources `yaml:"requests,omitempty"`
}
type ProcessResources struct {
NvidiaGPU string `yaml:"nvidia.com/gpu,omitempty"`
CPU string `yaml:"cpu,omitempty"`
Memory string `yaml:"memory,omitempty"`
}
type ProcessType string
const (
ProcessType_Cron ProcessType = "cron"
ProcessType_Job ProcessType = "job"
ProcessType_Web ProcessType = "web"
ProcessType_Worker ProcessType = "worker"
)
type ProcessCron struct {
ID string `yaml:"id"`
Schedule string `yaml:"schedule"`
Suffix string `yaml:"suffix"`
}
type ProcessPortMap struct {
ContainerPort int32 `yaml:"container_port"`
HostPort int32 `yaml:"host_port"`
Scheme string `yaml:"scheme"`
Protocol PortmapProtocol `yaml:"protocol"`
Name string `yaml:"name"`
}
type PortmapProtocol string
const (
PortmapProtocol_TCP PortmapProtocol = "TCP"
PortmapProtocol_UDP PortmapProtocol = "UDP"
)
type NameSorter []ProcessPortMap
func (a NameSorter) Len() int { return len(a) }
func (a NameSorter) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a NameSorter) Less(i, j int) bool { return a[i].Name < a[j].Name }
type ProcessTls struct {
Enabled bool `yaml:"enabled"`
IssuerName string `yaml:"issuer_name"`
} }
type ClusterIssuer struct { type ClusterIssuer struct {
@@ -303,459 +245,6 @@ type Job struct {
WorkingDir string WorkingDir string
} }
func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) {
if input.Schedule == "" {
return batchv1.CronJob{}, fmt.Errorf("Schedule cannot be empty")
}
labels := map[string]string{
"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
"app.kubernetes.io/name": input.ProcessType,
"app.kubernetes.io/part-of": input.AppName,
"dokku.com/cron-id": input.ID,
}
annotations := map[string]string{
"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
"dokku.com/builder-type": input.ImageSourceType,
"dokku.com/cron-id": input.ID,
"dokku.com/managed": "true",
}
for key, value := range input.Labels {
labels[key] = value
}
secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
env := []corev1.EnvVar{}
for key, value := range input.Env {
env = append(env, corev1.EnvVar{
Name: key,
Value: value,
})
}
suffix := input.Suffix
if suffix == "" {
n := 5
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
panic(err)
}
suffix = strings.ToLower(fmt.Sprintf("%X", b))
}
annotations["dokku.com/job-suffix"] = suffix
podAnnotations := annotations
podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)
job := batchv1.CronJob{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix),
Namespace: input.Namespace,
Labels: labels,
Annotations: annotations,
},
Spec: batchv1.CronJobSpec{
ConcurrencyPolicy: batchv1.AllowConcurrent,
FailedJobsHistoryLimit: ptr.To(int32(10)),
Schedule: input.Schedule,
StartingDeadlineSeconds: ptr.To(int64(60)),
SuccessfulJobsHistoryLimit: ptr.To(int32(10)),
Suspend: ptr.To(false),
TimeZone: ptr.To("Etc/UTC"),
JobTemplate: batchv1.JobTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
Annotations: annotations,
},
Spec: batchv1.JobSpec{
BackoffLimit: ptr.To(int32(0)),
PodReplacementPolicy: ptr.To(batchv1.Failed),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
Annotations: podAnnotations,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Args: input.Command,
Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
Env: env,
EnvFrom: []corev1.EnvFromSource{
{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: secretName,
},
Optional: ptr.To(true),
},
},
},
Image: input.Image,
ImagePullPolicy: corev1.PullAlways,
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
},
WorkingDir: input.WorkingDir,
},
},
RestartPolicy: corev1.RestartPolicyNever,
},
},
},
},
},
}
if input.Entrypoint != "" {
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command = []string{input.Entrypoint}
}
if input.RemoveContainer {
job.Spec.JobTemplate.Spec.TTLSecondsAfterFinished = ptr.To(int32(60))
}
if input.ImagePullSecrets != "" {
job.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
{
Name: input.ImagePullSecrets,
},
}
}
cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
if err != nil && cpuLimit != "" && cpuLimit != "0" {
cpuQuantity, err := resource.ParseQuantity(cpuLimit)
if err != nil {
return job, fmt.Errorf("Error parsing cpu limit: %w", err)
}
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
} else {
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = resource.MustParse("500m")
}
nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
if err != nil {
return job, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
}
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
}
memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
if err != nil && memoryLimit != "" && memoryLimit != "0" {
memoryQuantity, err := resource.ParseQuantity(memoryLimit)
if err != nil {
return job, fmt.Errorf("Error parsing memory limit: %w", err)
}
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
} else {
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = resource.MustParse("512Mi")
}
cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
if err != nil && cpuRequest != "" && cpuRequest != "0" {
cpuQuantity, err := resource.ParseQuantity(cpuRequest)
if err != nil {
return job, fmt.Errorf("Error parsing cpu request: %w", err)
}
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
} else {
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = resource.MustParse("500m")
}
memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
if err != nil && memoryRequest != "" && memoryRequest != "0" {
memoryQuantity, err := resource.ParseQuantity(memoryRequest)
if err != nil {
return job, fmt.Errorf("Error parsing memory request: %w", err)
}
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
} else {
job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = resource.MustParse("512Mi")
}
return job, nil
}
type Deployment struct {
AppName string
Command []string
Image string
ImagePullSecrets string
ImageSourceType string
Healthchecks []appjson.Healthcheck
Namespace string
PrimaryPort int32
PortMaps []PortMap
ProcessType string
Replicas int32
WorkingDir string
}
func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) {
labels := map[string]string{
"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
"app.kubernetes.io/name": input.ProcessType,
"app.kubernetes.io/part-of": input.AppName,
}
annotations := map[string]string{
"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
"dokku.com/builder-type": input.ImageSourceType,
"dokku.com/managed": "true",
}
secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
podAnnotations := annotations
podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)
deployment := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
Namespace: input.Namespace,
Labels: labels,
Annotations: annotations,
},
Spec: appsv1.DeploymentSpec{
Replicas: ptr.To(input.Replicas),
RevisionHistoryLimit: ptr.To(int32(5)),
Selector: &metav1.LabelSelector{
MatchLabels: labels,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
Annotations: podAnnotations,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
Env: []corev1.EnvVar{},
EnvFrom: []corev1.EnvFromSource{
{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: secretName,
},
Optional: ptr.To(true),
},
},
},
Image: input.Image,
ImagePullPolicy: corev1.PullAlways,
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
},
WorkingDir: input.WorkingDir,
},
},
},
},
},
}
if len(input.Command) > 0 {
deployment.Spec.Template.Spec.Containers[0].Args = input.Command
}
if len(input.Healthchecks) > 0 {
livenessChecks := []corev1.Probe{}
readinessChecks := []corev1.Probe{}
startupChecks := []corev1.Probe{}
uptimeSeconds := []int32{}
for _, healthcheck := range input.Healthchecks {
probe := corev1.Probe{
ProbeHandler: corev1.ProbeHandler{},
InitialDelaySeconds: healthcheck.InitialDelay,
PeriodSeconds: healthcheck.Wait,
TimeoutSeconds: healthcheck.Timeout,
FailureThreshold: healthcheck.Attempts,
SuccessThreshold: int32(1),
}
if len(healthcheck.Command) > 0 {
probe.ProbeHandler.Exec = &corev1.ExecAction{
Command: healthcheck.Command,
}
} else if healthcheck.Listening {
probe.ProbeHandler.TCPSocket = &corev1.TCPSocketAction{
Port: intstr.FromInt32(input.PrimaryPort),
}
for _, header := range healthcheck.HTTPHeaders {
if header.Name == "Host" {
probe.ProbeHandler.TCPSocket.Host = header.Value
}
}
} else if healthcheck.Path != "" {
probe.ProbeHandler.HTTPGet = &corev1.HTTPGetAction{
Path: healthcheck.Path,
Port: intstr.FromInt32(input.PrimaryPort),
HTTPHeaders: []corev1.HTTPHeader{},
}
if healthcheck.Scheme != "" {
probe.ProbeHandler.HTTPGet.Scheme = corev1.URIScheme(strings.ToUpper(healthcheck.Scheme))
}
for _, header := range healthcheck.HTTPHeaders {
probe.ProbeHandler.HTTPGet.HTTPHeaders = append(probe.ProbeHandler.HTTPGet.HTTPHeaders, corev1.HTTPHeader{
Name: header.Name,
Value: header.Value,
})
}
} else if healthcheck.Uptime > 0 {
uptimeSeconds = append(uptimeSeconds, healthcheck.Uptime)
}
if healthcheck.Type == appjson.HealthcheckType_Liveness {
livenessChecks = append(livenessChecks, probe)
} else if healthcheck.Type == appjson.HealthcheckType_Readiness {
readinessChecks = append(readinessChecks, probe)
} else if healthcheck.Type == appjson.HealthcheckType_Startup {
startupChecks = append(startupChecks, probe)
}
}
if len(livenessChecks) > 1 {
common.LogWarn("Multiple liveness checks are not supported, only the first one will be used")
}
if len(readinessChecks) > 1 {
common.LogWarn("Multiple readiness checks are not supported, only the first one will be used")
}
if len(startupChecks) > 1 {
common.LogWarn("Multiple startup checks are not supported, only the first one will be used")
}
if len(uptimeSeconds) > 1 {
common.LogWarn("Multiple uptime checks are not supported, only the first one will be used")
}
if len(livenessChecks) > 0 {
deployment.Spec.Template.Spec.Containers[0].LivenessProbe = &livenessChecks[0]
}
if len(readinessChecks) > 0 {
deployment.Spec.Template.Spec.Containers[0].ReadinessProbe = &readinessChecks[0]
}
if len(startupChecks) > 0 {
deployment.Spec.Template.Spec.Containers[0].StartupProbe = &startupChecks[0]
}
if len(uptimeSeconds) > 0 {
deployment.Spec.MinReadySeconds = uptimeSeconds[0]
}
}
if input.ProcessType == "web" {
for _, portMap := range input.PortMaps {
protocol := "TCP"
if portMap.Scheme == "udp" {
protocol = "UDP"
}
deployment.Spec.Template.Spec.Containers[0].Ports = append(deployment.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{
Name: portMap.String(),
ContainerPort: portMap.ContainerPort,
Protocol: corev1.Protocol(protocol),
})
}
deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
Name: "PORT",
Value: fmt.Sprint(input.PrimaryPort),
})
}
if input.ImagePullSecrets != "" {
deployment.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
{
Name: input.ImagePullSecrets,
},
}
}
cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...)
if err != nil && cpuLimit != "" && cpuLimit != "0" {
cpuQuantity, err := resource.ParseQuantity(cpuLimit)
if err != nil {
return deployment, fmt.Errorf("Error parsing cpu limit: %w", err)
}
deployment.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity
}
nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...)
if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit)
if err != nil {
return deployment, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
}
deployment.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity
}
memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...)
if err != nil && memoryLimit != "" && memoryLimit != "0" {
memoryQuantity, err := resource.ParseQuantity(memoryLimit)
if err != nil {
return deployment, fmt.Errorf("Error parsing memory limit: %w", err)
}
deployment.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity
}
cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...)
if err != nil && cpuRequest != "" && cpuRequest != "0" {
cpuQuantity, err := resource.ParseQuantity(cpuRequest)
if err != nil {
return deployment, fmt.Errorf("Error parsing cpu request: %w", err)
}
deployment.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity
}
memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...)
if err != nil && memoryRequest != "" && memoryRequest != "0" {
memoryQuantity, err := resource.ParseQuantity(memoryRequest)
if err != nil {
return deployment, fmt.Errorf("Error parsing memory request: %w", err)
}
deployment.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity
}
return deployment, nil
}
type IngressRouteEntrypoint string
const (
IngressRouteEntrypoint_HTTP IngressRouteEntrypoint = "web"
IngressRouteEntrypoint_HTTPS IngressRouteEntrypoint = "websecure"
)
type IngressRoute struct {
AppName string
Entrypoints []IngressRouteEntrypoint
Namespace string
PortMap PortMap
ProcessType string
ServiceName string
}
func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressRoute {
labels := map[string]string{
"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
"app.kubernetes.io/name": input.ProcessType,
"app.kubernetes.io/part-of": input.AppName,
}
annotations := map[string]string{
"dokku.com/managed": "true",
}
port := input.PortMap.String()
ingressRoute := traefikv1alpha1.IngressRoute{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", input.ServiceName, port),
Namespace: input.Namespace,
Labels: labels,
Annotations: annotations,
},
Spec: traefikv1alpha1.IngressRouteSpec{
EntryPoints: []string{string(IngressRouteEntrypoint_HTTP)},
},
}
return ingressRoute
}
func templateKubernetesJob(input Job) (batchv1.Job, error) { func templateKubernetesJob(input Job) (batchv1.Job, error) {
labels := map[string]string{ labels := map[string]string{
"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), "app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType),
@@ -907,137 +396,6 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
return job, nil return job, nil
} }
type Secret struct {
AppName string
Env map[string]string
Namespace string
}
func templateKubernetesSecret(input Secret) corev1.Secret {
secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName)
labels := map[string]string{
"app.kubernetes.io/instance": secretName,
"app.kubernetes.io/name": fmt.Sprintf("%s-env", input.AppName),
"app.kubernetes.io/part-of": input.AppName,
}
annotations := map[string]string{
"app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED",
"dokku.com/managed": "true",
}
secret := corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: input.Namespace,
Labels: labels,
Annotations: annotations,
},
Data: map[string][]byte{},
}
return secret
}
type Service struct {
AppName string
Namespace string
PortMaps []PortMap
}
func templateKubernetesService(input Service) corev1.Service {
labels := map[string]string{
"app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, "web"),
"app.kubernetes.io/name": "web",
"app.kubernetes.io/part-of": input.AppName,
}
annotations := map[string]string{
"dokku.com/managed": "true",
}
service := corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", input.AppName, "web"),
Namespace: input.Namespace,
Labels: labels,
Annotations: annotations,
},
Spec: corev1.ServiceSpec{
Selector: labels,
},
}
for _, portMap := range input.PortMaps {
protocol := "TCP"
if portMap.Scheme == "udp" {
protocol = "UDP"
}
service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{
Name: portMap.String(),
Port: portMap.HostPort,
TargetPort: intstr.FromString(portMap.String()),
Protocol: corev1.Protocol(protocol),
})
}
return service
}
type WriteResourceInput struct {
AppendContents string
Object runtime.Object
Path string
Replacements *orderedmap.OrderedMap[string, string]
}
func writeResourceToFile(input WriteResourceInput) error {
common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path))
printr := printers.NewTypeSetter(runtimeScheme).ToPrinter(&printers.YAMLPrinter{})
handle, err := os.Create(input.Path)
if err != nil {
return fmt.Errorf("Error creating template file: %w", err)
}
defer handle.Close()
if err := printr.PrintObj(input.Object, handle); err != nil {
return fmt.Errorf("Error writing template file: %w", err)
}
if input.Replacements != nil {
b, err := os.ReadFile(input.Path)
if err != nil {
return fmt.Errorf("Error reading template file: %w", err)
}
contents := string(b)
for pair := input.Replacements.Oldest(); pair != nil; pair = pair.Next() {
contents = strings.ReplaceAll(string(contents), pair.Key, pair.Value)
}
err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
if err != nil {
return fmt.Errorf("Error updating template file with replacements: %w", err)
}
}
if input.AppendContents != "" {
b, err := os.ReadFile(input.Path)
if err != nil {
return fmt.Errorf("Error reading template file: %w", err)
}
contents := string(b) + "\n" + input.AppendContents
err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644))
if err != nil {
return fmt.Errorf("Error updating template file with replacements: %w", err)
}
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(input.Path)
}
return nil
}
type WriteYamlInput struct { type WriteYamlInput struct {
Object interface{} Object interface{}
Path string Path string
@@ -1,4 +0,0 @@
dnsNames:
{{- range .Values.processes.PROCESS_TYPE.domains }}
- {{ . }}
{{- end }}
@@ -0,0 +1,32 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := .Values.processes.PROCESS_NAME }}
{{- if and $config.web.tls $config.domains }}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
annotations:
dokku.com/managed: "true"
creationTimestamp: null
labels:
app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
name: {{ $.Values.global.app_name }}-{{ $processName }}
namespace: {{ $.Values.global.namespace }}
spec:
issuerRef:
kind: ClusterIssuer
name: {{ $config.tls.issuer_name }}
secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
secretTemplate:
annotations:
dokku.com/managed: "true"
labels:
app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
status: {}
dnsNames:
{{- range $config.domains }}
- {{ . }}
{{- end }}
{{- end }}
@@ -0,0 +1,103 @@
{{- $processName := "CRON_ID" }}
{{- $config := .Values.processes.CRON_ID }}
---
apiVersion: batch/v1
kind: CronJob
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
dokku.com/builder-type: {{ $.Values.global.image.type }}
dokku.com/cron-id: {{ $config.cron.id }}
dokku.com/job-suffix: {{ $config.cron.suffix }}
dokku.com/managed: "true"
kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
app.kubernetes.io/name: cron
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
dokku.com/cron-id: {{ $config.cron.id }}
name: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
namespace: default
spec:
concurrencyPolicy: Allow
failedJobsHistoryLimit: 10
jobTemplate:
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
dokku.com/builder-type: {{ $.Values.global.image.type }}
dokku.com/cron-id: {{ $config.cron.id }}
dokku.com/job-suffix: {{ $config.cron.suffix }}
dokku.com/managed: "true"
kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
creationTimestamp: null
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
app.kubernetes.io/name: cron
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
dokku.com/cron-id: {{ $config.cron.id }}
spec:
backoffLimit: 0
podReplacementPolicy: Failed
template:
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
dokku.com/builder-type: {{ $.Values.global.image.type }}
dokku.com/cron-id: {{ $config.cron.id }}
dokku.com/job-suffix: {{ $config.cron.suffix }}
dokku.com/managed: "true"
kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron
creationTimestamp: null
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }}
app.kubernetes.io/name: cron
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
dokku.com/cron-id: {{ $config.cron.id }}
spec:
containers:
- args:
{{- range $config.args }}
- {{ . }}
{{- end }}
envFrom:
- secretRef:
name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
optional: true
image: {{ $.Values.global.image.name }}
imagePullPolicy: Always
name: {{ $.Values.global.app_name }}-cron
{{- if and $config.resources (or $config.resources.limits $config.resources.requests) }}
resources:
{{- if $config.resources.limits }}
limits:
{{- if $config.resources.limits.cpu }}
cpu: {{ $config.resources.limits.cpu }}
{{- end }}
{{- if $config.resources.limits.memory }}
memory: {{ $config.resources.limits.memory }}
{{- end }}
{{- end }}
{{- if $config.resources.requests }}
requests:
{{- if $config.resources.requests.cpu }}
cpu: {{ $config.resources.requests.cpu }}
{{- end }}
{{- if $config.resources.requests.memory }}
memory: {{ $config.resources.requests.memory }}
{{- end }}
{{- end }}
{{- end }}
{{- if $.Values.global.image.working_dir }}
workingDir: {{ $.Values.global.image.working_dir }}
{{- end }}
{{- if $.Values.global.image.image_pull_secrets }}
imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }}
{{- end }}
restartPolicy: Never
ttlSecondsAfterFinished: 60
schedule: {{ $config.cron.schedule }}
startingDeadlineSeconds: 60
successfulJobsHistoryLimit: 10
suspend: false
timeZone: Etc/UTC
@@ -0,0 +1,96 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := .Values.processes.PROCESS_NAME }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
dokku.com/builder-type: {{ $.Values.global.image.type }}
dokku.com/managed: "true"
kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
creationTimestamp: null
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
name: {{ $.Values.global.app_name }}-{{ $processName }}
namespace: default
spec:
replicas: {{ $config.replicas }}
revisionHistoryLimit: 5
selector:
matchLabels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
strategy: {}
template:
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
dokku.com/builder-type: {{ $.Values.global.image.type }}
dokku.com/managed: "true"
kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}
creationTimestamp: null
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
spec:
containers:
- args:
{{- range $config.args }}
- {{ . }}
{{- end }}
{{- if eq $processName "web" }}
env:
- name: PORT
value: "{{ $.Values.global.primary_port }}"
{{- end }}
envFrom:
- secretRef:
name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
optional: true
image: {{ $.Values.global.image.name }}
imagePullPolicy: Always
name: {{ $.Values.global.app_name }}-{{ $processName }}
{{- if eq $processName "web" }}
ports:
{{- range $pdx, $port_map := $config.web.port_maps }}
- containerPort: {{ $port_map.container_port }}
name: {{ $port_map.name }}
protocol: {{ $port_map.protocol }}
{{- end }}
{{- end }}
{{- if and $config.resources (or $config.resources.limits $config.resources.requests) }}
resources:
{{- if $config.resources.limits }}
limits:
{{- if $config.resources.limits.cpu }}
cpu: {{ $config.resources.limits.cpu }}
{{- end }}
{{- if $config.resources.limits.memory }}
memory: {{ $config.resources.limits.memory }}
{{- end }}
{{- end }}
{{- if $config.resources.requests }}
requests:
{{- if $config.resources.requests.cpu }}
cpu: {{ $config.resources.requests.cpu }}
{{- end }}
{{- if $config.resources.requests.memory }}
memory: {{ $config.resources.requests.memory }}
{{- end }}
{{- end }}
{{- end }}
{{- if $config.healthchecks.startup }}
startupProbe:
{{ $config.healthchecks.startup | toJson | indent 10 }}
{{- end }}
{{- if $.Values.global.image.working_dir }}
workingDir: {{ $.Values.global.image.working_dir }}
{{- end }}
{{- if $.Values.global.image.image_pull_secrets }}
imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }}
{{- end }}
@@ -0,0 +1,35 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := .Values.processes.PROCESS_NAME }}
{{- if $config.web.domains }}
{{- range $pdx, $port_map := $config.web.port_maps }}
---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
annotations:
dokku.com/managed: "true"
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }}
namespace: {{ $.Values.global.namespace }}
spec:
entryPoints:
- web
{{- if and $config.web.tls (eq $port_map "https") }}
- websecure
{{- end }}
routes:
{{- range $ddx, $domain := $config.web.domains }}
- kind: Rule
match: Host(`{{ $domain }}`)
services:
- name: {{ $.Values.global.app_name }}-{{ $processName }}
namespace: {{ $.Values.global.namespace }}
passHostHeader: true
port: {{ $port_map.name }}
scheme: {{ $port_map.scheme }}
{{- end }}
{{- end }}
{{- end }}
@@ -0,0 +1,16 @@
apiVersion: v1
kind: Secret
metadata:
annotations:
app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }}
dokku.com/managed: "true"
labels:
app.kubernetes.io/instance: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
app.kubernetes.io/name: env-{{ $.Values.global.app_name }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }}
namespace: {{ $.Values.global.namespace }}
{{- with .Values.global.secrets }}
data:
{{- toYaml . | nindent 2 }}
{{- end }}
@@ -0,0 +1,26 @@
{{- $processName := "PROCESS_NAME" }}
{{- $config := .Values.processes.PROCESS_NAME }}
---
apiVersion: v1
kind: Service
metadata:
annotations:
dokku.com/managed: "true"
labels:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
name: {{ $.Values.global.app_name }}-{{ $processName }}
namespace: default
spec:
ports:
{{- range $pdx, $port_map := $config.web.port_maps }}
- name: {{ $port_map.name }}
port: {{ $port_map.host_port }}
protocol: TCP
targetPort: {{ $port_map.name }}
{{- end }}
selector:
app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }}
app.kubernetes.io/name: {{ $processName }}
app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
@@ -1,18 +0,0 @@
{{ if .Values.processes.PROCESS_TYPE.tls }}
- websecure
{{- end }}
routes:
{{- range .Values.processes.PROCESS_TYPE.domains }}
- kind: Rule
match: Host(`{{ . }}`)
services:
- name: APP_NAME-PROCESS_TYPE
namespace: NAMESPACE
passHostHeader: true
port: PORT_MAPPING
scheme: PORT_SCHEME
{{- end }}
{{ if .Values.processes.PROCESS_TYPE.tls }}
tls:
secretName: tls-APP_NAME-PROCESS_TYPE
{{- end }}
+179 -188
View File
@@ -3,6 +3,7 @@ package scheduler_k3s
import ( import (
"bufio" "bufio"
"context" "context"
"crypto/rand"
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"errors" "errors"
@@ -26,9 +27,7 @@ import (
"github.com/kballard/go-shellquote" "github.com/kballard/go-shellquote"
"github.com/rancher/wharfie/pkg/registries" "github.com/rancher/wharfie/pkg/registries"
"github.com/ryanuber/columnize" "github.com/ryanuber/columnize"
orderedmap "github.com/wk8/go-ordered-map/v2"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/client/conditions" "k8s.io/kubernetes/pkg/client/conditions"
@@ -227,34 +226,26 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
if err != nil { if err != nil {
return fmt.Errorf("Error creating chart directory: %w", err) return fmt.Errorf("Error creating chart directory: %w", err)
} }
defer os.RemoveAll(chartDir) // defer os.RemoveAll(chartDir)
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil { if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
return fmt.Errorf("Error creating chart templates directory: %w", err) return fmt.Errorf("Error creating chart templates directory: %w", err)
} }
deploymentId := time.Now().Unix() deploymentId := time.Now().Unix()
replacements := orderedmap.New[string, string]() b, err := templates.ReadFile("templates/chart/secret.yaml")
replacements.Set("DEPLOYMENT_ID_QUOTED", "{{.Values.deploment_id | quote}}")
replacements.Set("DEPLOYMENT_ID", "{{.Values.deploment_id}}")
secret := templateKubernetesSecret(Secret{
AppName: appName,
Env: env.Map(),
Namespace: namespace,
})
err = writeResourceToFile(WriteResourceInput{
Object: &secret,
Path: filepath.Join(chartDir, "templates/secret.yaml"),
Replacements: replacements,
AppendContents: `{{- with .Values.secrets }}
data:
{{- toYaml . | nindent 2 }}
{{- end }}
`,
})
if err != nil { if err != nil {
return fmt.Errorf("Error printing deployment: %w", err) return fmt.Errorf("Error reading secret template: %w", err)
}
filename := filepath.Join(chartDir, "templates", "secrets.yaml")
err = os.WriteFile(filename, b, os.FileMode(0644))
if err != nil {
return fmt.Errorf("Error writing secrets template: %w", err)
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(filename)
} }
portMaps, err := getPortMaps(appName) portMaps, err := getPortMaps(appName)
@@ -276,125 +267,12 @@ data:
} }
workingDir := common.GetWorkingDir(appName, image) workingDir := common.GetWorkingDir(appName, image)
deployments := map[string]appsv1.Deployment{}
i := 0
for processType := range processes {
startCommand, err := getStartCommand(StartCommandInput{
AppName: appName,
ProcessType: processType,
ImageSourceType: imageSourceType,
Port: primaryPort,
Env: env.Map(),
})
if err != nil {
return fmt.Errorf("Error getting start command for deployment: %w", err)
}
i++
replicaCountPlaceholder := int32(i * 1000)
healthchecks, ok := appJSON.Healthchecks[processType]
if !ok {
healthchecks = []appjson.Healthcheck{}
}
// todo: implement deployment annotations
// todo: implement pod annotations
// todo: implement volumes
deployment, err := templateKubernetesDeployment(Deployment{
AppName: appName,
Command: startCommand.Command,
Image: image,
ImagePullSecrets: imagePullSecrets,
ImageSourceType: imageSourceType,
Healthchecks: healthchecks,
Namespace: namespace,
PrimaryPort: primaryPort,
PortMaps: portMaps,
ProcessType: processType,
Replicas: replicaCountPlaceholder,
WorkingDir: workingDir,
})
if err != nil {
return fmt.Errorf("Error templating deployment: %w", err)
}
replacements.Set(fmt.Sprintf("replicas: %d", replicaCountPlaceholder), fmt.Sprintf("replicas: {{.Values.processes.%s.replicas}}", processType))
deployments[processType] = deployment
err = writeResourceToFile(WriteResourceInput{
Object: &deployment,
Path: filepath.Join(chartDir, fmt.Sprintf("templates/deployment-%s.yaml", deployment.Name)),
Replacements: replacements,
})
if err != nil {
return fmt.Errorf("Error printing deployment: %w", err)
}
replacements.Delete(fmt.Sprintf("replicas: %d", replicaCountPlaceholder))
}
cronEntries, err := cron.FetchCronEntries(appName) cronEntries, err := cron.FetchCronEntries(appName)
if err != nil { if err != nil {
return fmt.Errorf("Error fetching cron entries: %w", err) return fmt.Errorf("Error fetching cron entries: %w", err)
} }
clientset, err := NewKubernetesClient()
if err != nil {
return fmt.Errorf("Error creating kubernetes client: %w", err)
}
cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
Namespace: namespace,
})
if err != nil {
return fmt.Errorf("Error listing cron jobs: %w", err)
}
for _, cronEntry := range cronEntries {
suffix := ""
for _, cronJob := range cronJobs {
if cronJob.Labels["dokku.com/cron-id"] == cronEntry.ID {
var ok bool
suffix, ok = cronJob.Annotations["dokku.com/job-suffix"]
if !ok {
suffix = ""
}
}
}
words, err := shellquote.Split(cronEntry.Command)
if err != nil {
return fmt.Errorf("Error parsing cron command: %w", err)
}
cronJob, err := templateKubernetesCronJob(Job{
AppName: appName,
Command: words,
Env: map[string]string{},
ID: cronEntry.ID,
Image: image,
ImagePullSecrets: imagePullSecrets,
ImageSourceType: imageSourceType,
Namespace: namespace,
ProcessType: "cron",
Schedule: cronEntry.Schedule,
Suffix: suffix,
WorkingDir: workingDir,
})
if err != nil {
return fmt.Errorf("Error templating cron job: %w", err)
}
err = writeResourceToFile(WriteResourceInput{
Object: &cronJob,
Path: filepath.Join(chartDir, fmt.Sprintf("templates/cron-job-%s.yaml", cronEntry.ID)),
Replacements: replacements,
})
if err != nil {
return fmt.Errorf("Error printing cron job: %w", err)
}
}
issuerName := "letsencrypt-stag" issuerName := "letsencrypt-stag"
server := getComputedLetsencryptServer(appName) server := getComputedLetsencryptServer(appName)
if server == "prod" || server == "production" { if server == "prod" || server == "production" {
@@ -412,22 +290,7 @@ data:
} }
domains := []string{} domains := []string{}
if deployment, ok := deployments["web"]; ok { if _, ok := processes["web"]; ok {
service := templateKubernetesService(Service{
AppName: appName,
Namespace: namespace,
PortMaps: portMaps,
})
err := writeResourceToFile(WriteResourceInput{
Object: &service,
Path: filepath.Join(chartDir, "templates/service-web.yaml"),
Replacements: replacements,
})
if err != nil {
return fmt.Errorf("Error printing service: %w", err)
}
err = common.PlugnTrigger("domains-vhost-enabled", []string{appName}...) err = common.PlugnTrigger("domains-vhost-enabled", []string{appName}...)
if err == nil { if err == nil {
b, err := common.PlugnTriggerOutput("domains-list", []string{appName}...) b, err := common.PlugnTriggerOutput("domains-list", []string{appName}...)
@@ -442,34 +305,6 @@ data:
} }
} }
} }
err = createIngressRoutesFiles(CreateIngressRoutesInput{
AppName: appName,
ChartDir: chartDir,
Deployment: deployment,
Namespace: namespace,
PortMaps: portMaps,
ProcessType: "web",
Service: service,
})
if err != nil {
return fmt.Errorf("Error creating ingress routes: %w", err)
}
err = createCertificateFile(CreateCertificateFileInput{
ChartDir: chartDir,
Certificate: Certificate{
AppName: appName,
Name: fmt.Sprintf("%s-%s", appName, "web"),
Namespace: namespace,
TLS: tls,
},
IssuerName: issuerName,
ProcessType: "web",
})
if err != nil {
return fmt.Errorf("Error creating certificate files: %w", err)
}
} }
chart := &Chart{ chart := &Chart{
@@ -488,25 +323,181 @@ data:
} }
values := &Values{ values := &Values{
DeploymentID: fmt.Sprint(deploymentId), Global: GlobalValues{
Secrets: map[string]string{}, AppName: appName,
Processes: map[string]ProcessValues{}, DeploymentID: fmt.Sprint(deploymentId),
Image: GlobalImage{
ImagePullSecrets: imagePullSecrets,
Name: image,
Type: imageSourceType,
WorkingDir: workingDir,
},
Namespace: namespace,
PrimaryPort: primaryPort,
Secrets: map[string]string{},
},
Processes: map[string]ProcessValues{},
} }
for processType, processCount := range processes { for processType, processCount := range processes {
// todo: implement deployment annotations
// todo: implement pod annotations
// todo: implement volumes
healthchecks, ok := appJSON.Healthchecks[processType]
if !ok {
healthchecks = []appjson.Healthcheck{}
}
processHealthchecks := getProcessHealtchecks(healthchecks, primaryPort)
startCommand, err := getStartCommand(StartCommandInput{
AppName: appName,
ProcessType: processType,
ImageSourceType: imageSourceType,
Port: primaryPort,
Env: env.Map(),
})
if err != nil {
return fmt.Errorf("Error getting start command for deployment: %w", err)
}
args := startCommand.Command
processResources, err := getProcessResources(appName, processType)
if err != nil {
return fmt.Errorf("Error getting process resources: %w", err)
}
processValues := ProcessValues{ processValues := ProcessValues{
Replicas: int32(processCount), Args: args,
Healthchecks: processHealthchecks,
ProcessType: ProcessType_Worker,
Replicas: int32(processCount),
Resources: processResources,
} }
if processType == "web" { if processType == "web" {
sort.Strings(domains) processValues.Web = ProcessWeb{
processValues.Domains = domains Domains: domains,
processValues.TLS = tls PortMaps: []ProcessPortMap{},
TLS: tls,
}
processValues.ProcessType = ProcessType_Web
for _, portMap := range portMaps {
protocol := PortmapProtocol_TCP
if portMap.Scheme == "udp" {
protocol = PortmapProtocol_UDP
}
processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
ContainerPort: portMap.ContainerPort,
HostPort: portMap.HostPort,
Name: portMap.String(),
Protocol: protocol,
Scheme: portMap.Scheme,
})
}
sort.Sort(NameSorter(processValues.Web.PortMaps))
sort.Strings(processValues.Web.Domains)
} }
values.Processes[processType] = processValues values.Processes[processType] = processValues
for _, templateName := range []string{"deployment", "service", "certificate", "ingress-route"} {
b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
if err != nil {
return fmt.Errorf("Error reading %s template: %w", templateName, err)
}
filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s-%s.yaml", templateName, processType))
contents := strings.ReplaceAll(string(b), "PROCESS_NAME", processType)
err = os.WriteFile(filename, []byte(contents), os.FileMode(0644))
if err != nil {
return fmt.Errorf("Error writing %s template: %w", templateName, err)
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(filename)
}
}
}
clientset, err := NewKubernetesClient()
if err != nil {
return fmt.Errorf("Error creating kubernetes client: %w", err)
}
cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
Namespace: namespace,
})
if err != nil {
return fmt.Errorf("Error listing cron jobs: %w", err)
}
for _, cronEntry := range cronEntries {
// todo: implement deployment annotations
// todo: implement pod annotations
// todo: implement volumes
suffix := ""
for _, cronJob := range cronJobs {
if cronJob.Labels["dokku.com/cron-id"] == cronEntry.ID {
var ok bool
suffix, ok = cronJob.Annotations["dokku.com/job-suffix"]
if !ok {
suffix = ""
}
}
}
if suffix == "" {
n := 5
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
panic(err)
}
suffix = strings.ToLower(fmt.Sprintf("%X", b))
}
words, err := shellquote.Split(cronEntry.Command)
if err != nil {
return fmt.Errorf("Error parsing cron command: %w", err)
}
processResources, err := getProcessResources(appName, cronEntry.ID)
if err != nil {
return fmt.Errorf("Error getting process resources: %w", err)
}
processValues := ProcessValues{
Args: words,
Cron: ProcessCron{
ID: cronEntry.ID,
Schedule: cronEntry.Schedule,
Suffix: suffix,
},
ProcessType: ProcessType_Cron,
Replicas: 1,
Resources: processResources,
}
values.Processes[cronEntry.ID] = processValues
b, err := templates.ReadFile("templates/chart/cron-job.yaml")
if err != nil {
return fmt.Errorf("Error reading cron job template: %w", err)
}
cronFile := filepath.Join(chartDir, "templates", fmt.Sprintf("cron-job-%s.yaml", cronEntry.ID))
contents := strings.ReplaceAll(string(b), "CRON_ID", cronEntry.ID)
err = os.WriteFile(cronFile, []byte(contents), os.FileMode(0644))
if err != nil {
return fmt.Errorf("Error writing cron job template: %w", err)
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(cronFile)
}
} }
for key, value := range env.Map() { for key, value := range env.Map() {
values.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value)) values.Global.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value))
} }
err = writeYaml(WriteYamlInput{ err = writeYaml(WriteYamlInput{