diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 9f43ec8bc..e7387bbac 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -10,10 +10,12 @@ import ( "strings" "time" + appjson "github.com/dokku/dokku/plugins/app-json" "github.com/dokku/dokku/plugins/common" "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" @@ -355,6 +357,151 @@ func getGlobalGlobalToken() string { return common.PropertyGet("scheduler-k3s", "--global", "token") } +func getProcessHealtchecks(healthchecks []appjson.Healthcheck, primaryPort int32) ProcessHealthchecks { + if len(healthchecks) == 0 { + return ProcessHealthchecks{} + } + + livenessChecks := []ProcessHealthcheck{} + readinessChecks := []ProcessHealthcheck{} + startupChecks := []ProcessHealthcheck{} + uptimeSeconds := []int32{} + for _, healthcheck := range healthchecks { + probe := ProcessHealthcheck{ + InitialDelaySeconds: healthcheck.InitialDelay, + PeriodSeconds: healthcheck.Wait, + TimeoutSeconds: healthcheck.Timeout, + FailureThreshold: healthcheck.Attempts, + SuccessThreshold: int32(1), + } + if len(healthcheck.Command) > 0 { + probe.Exec = &ExecHealthcheck{ + Command: healthcheck.Command, + } + } else if healthcheck.Listening { + probe.TCPSocket = &TCPHealthcheck{ + Port: primaryPort, + } + for _, header := range healthcheck.HTTPHeaders { + if header.Name == "Host" { + probe.TCPSocket.Host = header.Value + } + } + } else if healthcheck.Path != "" { + probe.HTTPGet = &HTTPHealthcheck{ + Path: healthcheck.Path, + Port: primaryPort, + HTTPHeaders: []HTTPHeader{}, + } + + if healthcheck.Scheme != "" { + probe.HTTPGet.Scheme = URIScheme(strings.ToUpper(healthcheck.Scheme)) + } + + for _, header := range healthcheck.HTTPHeaders { + probe.HTTPGet.HTTPHeaders = append(probe.HTTPGet.HTTPHeaders, HTTPHeader{ + Name: header.Name, + Value: header.Value, + }) + } + } else if healthcheck.Uptime > 0 { + uptimeSeconds = append(uptimeSeconds, healthcheck.Uptime) + } + + if healthcheck.Type == appjson.HealthcheckType_Liveness { + livenessChecks = append(livenessChecks, probe) + } else if healthcheck.Type == appjson.HealthcheckType_Readiness { + readinessChecks = append(readinessChecks, probe) + } else if healthcheck.Type == appjson.HealthcheckType_Startup { + startupChecks = append(startupChecks, probe) + } + } + if len(livenessChecks) > 1 { + common.LogWarn("Multiple liveness checks are not supported, only the first one will be used") + } + if len(readinessChecks) > 1 { + common.LogWarn("Multiple readiness checks are not supported, only the first one will be used") + } + if len(startupChecks) > 1 { + common.LogWarn("Multiple startup checks are not supported, only the first one will be used") + } + if len(uptimeSeconds) > 1 { + common.LogWarn("Multiple uptime checks are not supported, only the first one will be used") + } + + processHealthchecks := ProcessHealthchecks{} + if len(livenessChecks) > 0 { + processHealthchecks.Liveness = livenessChecks[0] + } + if len(readinessChecks) > 0 { + processHealthchecks.Readiness = readinessChecks[0] + } + if len(startupChecks) > 0 { + processHealthchecks.Startup = startupChecks[0] + } + if len(uptimeSeconds) > 0 { + processHealthchecks.MinReadySeconds = uptimeSeconds[0] + } + + return processHealthchecks +} + +func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) { + processResources := ProcessResourcesMap{ + Limits: ProcessResources{ + CPU: "1000m", + Memory: "512Mi", + }, + Requests: ProcessResources{ + CPU: "1000m", + Memory: "512Mi", + }, + } + cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "cpu"}...) + if err != nil && cpuLimit != "" && cpuLimit != "0" { + _, err := resource.ParseQuantity(cpuLimit) + if err != nil { + return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err) + } + processResources.Limits.CPU = cpuLimit + } + nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...) + if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" { + _, err := resource.ParseQuantity(nvidiaGpuLimit) + if err != nil { + return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err) + } + processResources.Limits.NvidiaGPU = nvidiaGpuLimit + } + memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "memory"}...) + if err != nil && memoryLimit != "" && memoryLimit != "0" { + _, err := resource.ParseQuantity(memoryLimit) + if err != nil { + return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err) + } + processResources.Limits.Memory = memoryLimit + } + + cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "cpu"}...) + if err != nil && cpuRequest != "" && cpuRequest != "0" { + _, err := resource.ParseQuantity(cpuRequest) + if err != nil { + return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err) + } + processResources.Requests.CPU = cpuRequest + } + memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "memory"}...) + if err != nil && memoryRequest != "" && memoryRequest != "0" { + _, err := resource.ParseQuantity(memoryRequest) + if err != nil { + return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err) + } + processResources.Requests.Memory = memoryRequest + } + + return processResources, nil +} + func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { command := extractStartCommand(input) fields, err := shell.Fields(command, func(name string) string { @@ -407,7 +554,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient) error { } } - contents, err := templates.ReadFile(fmt.Sprintf("templates/%s.yaml", chart.ReleaseName)) + contents, err := templates.ReadFile(fmt.Sprintf("templates/helm-config/%s.yaml", chart.ReleaseName)) if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Error reading values file %s: %w", chart.ReleaseName, err) } diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 8faeac9fa..ee1a2b0c2 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -258,7 +258,7 @@ func CommandInitialize(taintScheduling bool) error { } common.LogInfo2Quiet("Updating traefik config") - contents, err := templates.ReadFile("templates/traefik-config.yaml") + contents, err := templates.ReadFile("templates/helm-config/traefik-config.yaml") if err != nil { return fmt.Errorf("Unable to read traefik config template: %w", err) } diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 3ea21ef1f..845ebe3e7 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -1,30 +1,20 @@ package scheduler_k3s import ( - "bytes" "crypto/rand" "fmt" "os" - "path/filepath" "strings" acmev1 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" - appjson "github.com/dokku/dokku/plugins/app-json" "github.com/dokku/dokku/plugins/common" - traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1" - orderedmap "github.com/wk8/go-ordered-map/v2" "gopkg.in/yaml.v3" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/cli-runtime/pkg/printers" "k8s.io/utils/ptr" ) @@ -36,189 +26,141 @@ type Chart struct { } type Values struct { - DeploymentID string `yaml:"deploment_id"` - Secrets map[string]string `yaml:"secrets,omitempty"` - Processes map[string]ProcessValues `yaml:"processes"` + Global GlobalValues `yaml:"global"` + Processes map[string]ProcessValues `yaml:"processes"` +} + +type GlobalValues struct { + AppName string `yaml:"app_name"` + DeploymentID string `yaml:"deploment_id"` + Image GlobalImage `yaml:"image"` + Namespace string `yaml:"namespace"` + PrimaryPort int32 `yaml:"primary_port"` + Secrets map[string]string `yaml:"secrets,omitempty"` +} + +type GlobalImage struct { + ImagePullSecrets string `yaml:"image_pull_secrets"` + Name string `yaml:"name"` + Type string `yaml:"type"` + WorkingDir string `yaml:"working_dir"` } type ProcessValues struct { - Domains []string `yaml:"domains,omitempty"` - Replicas int32 `yaml:"replicas"` - TLS bool `yaml:"tls"` + Args []string `yaml:"args,omitempty"` + Cron ProcessCron `yaml:"cron,omitempty"` + Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"` + ProcessType ProcessType `yaml:"process_type"` + Replicas int32 `yaml:"replicas"` + Resources ProcessResourcesMap `yaml:"resources,omitempty"` + Web ProcessWeb `yaml:"web,omitempty"` } -type CreateIngressRoutesInput struct { - AppName string - ChartDir string - Deployment appsv1.Deployment - Namespace string - ProcessType string - PortMaps []PortMap - Service v1.Service +type ProcessHealthchecks struct { + Liveness ProcessHealthcheck `yaml:"liveness,omitempty"` + Readiness ProcessHealthcheck `yaml:"readiness,omitempty"` + Startup ProcessHealthcheck `yaml:"startup,omitempty"` + MinReadySeconds int32 `yaml:"min_ready_seconds,omitempty"` } -func createIngressRoutesFiles(input CreateIngressRoutesInput) error { - for _, portMap := range input.PortMaps { - ingressRoute := templateKubernetesIngressRoute(IngressRoute{ - AppName: input.AppName, - Namespace: input.Namespace, - PortMap: portMap, - ProcessType: input.ProcessType, - ServiceName: input.Service.Name, - }) +type ProcessHealthcheck struct { + Exec *ExecHealthcheck `yaml:"exec,omitempty"` + HTTPGet *HTTPHealthcheck `yaml:"httpGet,omitempty"` + TCPSocket *TCPHealthcheck `yaml:"tcpSocket,omitempty"` - ingressRouteFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/ingress-route-%s.yaml", portMap.String())) - err := writeResourceToFile(WriteResourceInput{ - Object: &ingressRoute, - Path: ingressRouteFile, - }) - if err != nil { - return fmt.Errorf("Error printing ingress route: %w", err) - } - - b, err := os.ReadFile(ingressRouteFile) - if err != nil { - return fmt.Errorf("Error reading ingress route file: %w", err) - } - - append, err := templates.ReadFile("templates/ingress-routes-append.yaml") - if err != nil { - return fmt.Errorf("Error reading ingress route append file: %w", err) - } - - contents := strings.Join([]string{strings.TrimSpace(string(b)), string(append)}, "") - contents = strings.ReplaceAll(contents, " routes: null", "") - contents = strings.Join([]string{ - "{{- if .Values.processes.PROCESS_TYPE.domains }}", - contents, - "{{- end }}", - }, "\n") - - replacements := orderedmap.New[string, string]() - replacements.Set("PROCESS_TYPE", input.ProcessType) - replacements.Set("APP_NAME", input.AppName) - replacements.Set("NAMESPACE", input.Namespace) - replacements.Set("PORT_MAPPING", portMap.String()) - replacements.Set("PORT_SCHEME", portMap.Scheme) - for pair := replacements.Oldest(); pair != nil; pair = pair.Next() { - contents = strings.ReplaceAll(contents, pair.Key, pair.Value) - } - - err = os.WriteFile(ingressRouteFile, []byte(contents), os.FileMode(0644)) - if err != nil { - return fmt.Errorf("Error writing ingress route file: %w", err) - } - - if os.Getenv("DOKKU_TRACE") == "1" { - common.CatFile(ingressRouteFile) - } - } - - return nil + InitialDelaySeconds int32 `yaml:"initialDelaySeconds,omitempty"` + TimeoutSeconds int32 `yaml:"timeoutSeconds,omitempty"` + PeriodSeconds int32 `yaml:"periodSeconds,omitempty"` + SuccessThreshold int32 `yaml:"successThreshold,omitempty"` + FailureThreshold int32 `yaml:"failureThreshold,omitempty"` + TerminationGracePeriodSeconds *int64 `yaml:"terminationGracePeriodSeconds,omitempty"` } -type CreateCertificateFileInput struct { - Certificate Certificate - ChartDir string - IssuerName string - ProcessType string +type ExecHealthcheck struct { + Command []string `yaml:"command,omitempty"` } -type Certificate struct { - AppName string - Name string - Namespace string - TLS bool +type HTTPHealthcheck struct { + Path string `yaml:"path,omitempty"` + Port int32 `yaml:"port,omitempty"` + Host string `yaml:"host,omitempty"` + Scheme URIScheme `yaml:"scheme,omitempty"` + HTTPHeaders []HTTPHeader `yaml:"httpHeaders,omitempty"` } -func createCertificateFile(input CreateCertificateFileInput) error { - certificate, err := templateKubernetesCertificate(input.Certificate) - if err != nil { - return fmt.Errorf("Error templating certificate: %w", err) - } - - certificateFile := filepath.Join(input.ChartDir, fmt.Sprintf("templates/certificate-%s.yaml", input.ProcessType)) - err = writeResourceToFile(WriteResourceInput{ - Object: &certificate, - Path: certificateFile, - }) - if err != nil { - return fmt.Errorf("Error printing ingress route: %w", err) - } - - b, err := os.ReadFile(certificateFile) - if err != nil { - return fmt.Errorf("Error reading ingress route file: %w", err) - } - - append, err := templates.ReadFile("templates/certificate-append.yaml") - if err != nil { - return fmt.Errorf("Error reading ingress route append file: %w", err) - } - - contents := string(bytes.Join([][]byte{b, append}, []byte(""))) - contents = strings.Join([]string{ - "{{- if and .Values.processes.PROCESS_TYPE.tls .Values.processes.PROCESS_TYPE.domains }}", - contents, - "{{- end }}", - }, "\n") - - replacements := orderedmap.New[string, string]() - replacements.Set("PROCESS_TYPE", input.ProcessType) - replacements.Set("ISSUER_NAME", input.IssuerName) - for pair := replacements.Oldest(); pair != nil; pair = pair.Next() { - contents = strings.ReplaceAll(contents, pair.Key, pair.Value) - } - - err = os.WriteFile(certificateFile, []byte(contents), os.FileMode(0644)) - if err != nil { - return fmt.Errorf("Error writing ingress route file: %w", err) - } - - if os.Getenv("DOKKU_TRACE") == "1" { - common.CatFile(certificateFile) - } - - return nil +type TCPHealthcheck struct { + Port int32 `yaml:"port,omitempty"` + Host string `yaml:"host,omitempty"` } -func templateKubernetesCertificate(input Certificate) (certmanagerv1.Certificate, error) { - if input.Name == "" { - return certmanagerv1.Certificate{}, fmt.Errorf("Name cannot be empty") - } - if input.Namespace == "" { - return certmanagerv1.Certificate{}, fmt.Errorf("Namespace cannot be empty") - } +type HTTPHeader struct { + Name string `yaml:"name"` + Value string `yaml:"value"` +} - labels := map[string]string{ - "app.kubernetes.io/name": input.Name, - "app.kubernetes.io/part-of": input.AppName, - } - annotations := map[string]string{ - "dokku.com/managed": "true", - } +type URIScheme string - certificate := certmanagerv1.Certificate{ - ObjectMeta: metav1.ObjectMeta{ - Name: input.Name, - Namespace: input.Namespace, - Labels: labels, - Annotations: annotations, - }, - Spec: certmanagerv1.CertificateSpec{ - SecretName: "tls-" + input.Name, - SecretTemplate: &certmanagerv1.CertificateSecretTemplate{ - Annotations: annotations, - Labels: labels, - }, - IssuerRef: certmanagermetav1.ObjectReference{ - Name: "ISSUER_NAME", - Kind: "ClusterIssuer", - }, - }, - } +const ( + URISchemeHTTP URIScheme = "HTTP" + URISchemeHTTPS URIScheme = "HTTPS" +) - return certificate, nil +type ProcessWeb struct { + Domains []string `yaml:"domains,omitempty"` + PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"` + TLS bool `yaml:"tls"` +} + +type ProcessResourcesMap struct { + Limits ProcessResources `yaml:"limits,omitempty"` + Requests ProcessResources `yaml:"requests,omitempty"` +} + +type ProcessResources struct { + NvidiaGPU string `yaml:"nvidia.com/gpu,omitempty"` + CPU string `yaml:"cpu,omitempty"` + Memory string `yaml:"memory,omitempty"` +} + +type ProcessType string + +const ( + ProcessType_Cron ProcessType = "cron" + ProcessType_Job ProcessType = "job" + ProcessType_Web ProcessType = "web" + ProcessType_Worker ProcessType = "worker" +) + +type ProcessCron struct { + ID string `yaml:"id"` + Schedule string `yaml:"schedule"` + Suffix string `yaml:"suffix"` +} + +type ProcessPortMap struct { + ContainerPort int32 `yaml:"container_port"` + HostPort int32 `yaml:"host_port"` + Scheme string `yaml:"scheme"` + Protocol PortmapProtocol `yaml:"protocol"` + Name string `yaml:"name"` +} + +type PortmapProtocol string + +const ( + PortmapProtocol_TCP PortmapProtocol = "TCP" + PortmapProtocol_UDP PortmapProtocol = "UDP" +) + +type NameSorter []ProcessPortMap + +func (a NameSorter) Len() int { return len(a) } +func (a NameSorter) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a NameSorter) Less(i, j int) bool { return a[i].Name < a[j].Name } + +type ProcessTls struct { + Enabled bool `yaml:"enabled"` + IssuerName string `yaml:"issuer_name"` } type ClusterIssuer struct { @@ -303,459 +245,6 @@ type Job struct { WorkingDir string } -func templateKubernetesCronJob(input Job) (batchv1.CronJob, error) { - if input.Schedule == "" { - return batchv1.CronJob{}, fmt.Errorf("Schedule cannot be empty") - } - labels := map[string]string{ - "app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), - "app.kubernetes.io/name": input.ProcessType, - "app.kubernetes.io/part-of": input.AppName, - "dokku.com/cron-id": input.ID, - } - annotations := map[string]string{ - "app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED", - "dokku.com/builder-type": input.ImageSourceType, - "dokku.com/cron-id": input.ID, - "dokku.com/managed": "true", - } - - for key, value := range input.Labels { - labels[key] = value - } - secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName) - - env := []corev1.EnvVar{} - for key, value := range input.Env { - env = append(env, corev1.EnvVar{ - Name: key, - Value: value, - }) - } - - suffix := input.Suffix - if suffix == "" { - n := 5 - b := make([]byte, n) - if _, err := rand.Read(b); err != nil { - panic(err) - } - suffix = strings.ToLower(fmt.Sprintf("%X", b)) - } - annotations["dokku.com/job-suffix"] = suffix - - podAnnotations := annotations - podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType) - - job := batchv1.CronJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix), - Namespace: input.Namespace, - Labels: labels, - Annotations: annotations, - }, - Spec: batchv1.CronJobSpec{ - ConcurrencyPolicy: batchv1.AllowConcurrent, - FailedJobsHistoryLimit: ptr.To(int32(10)), - Schedule: input.Schedule, - StartingDeadlineSeconds: ptr.To(int64(60)), - SuccessfulJobsHistoryLimit: ptr.To(int32(10)), - Suspend: ptr.To(false), - TimeZone: ptr.To("Etc/UTC"), - JobTemplate: batchv1.JobTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: annotations, - }, - Spec: batchv1.JobSpec{ - BackoffLimit: ptr.To(int32(0)), - PodReplacementPolicy: ptr.To(batchv1.Failed), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: podAnnotations, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Args: input.Command, - Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), - Env: env, - EnvFrom: []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, - }, - Optional: ptr.To(true), - }, - }, - }, - Image: input.Image, - ImagePullPolicy: corev1.PullAlways, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{}, - Requests: corev1.ResourceList{}, - }, - WorkingDir: input.WorkingDir, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - }, - }, - }, - } - - if input.Entrypoint != "" { - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command = []string{input.Entrypoint} - } - - if input.RemoveContainer { - job.Spec.JobTemplate.Spec.TTLSecondsAfterFinished = ptr.To(int32(60)) - } - - if input.ImagePullSecrets != "" { - job.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{ - { - Name: input.ImagePullSecrets, - }, - } - } - - cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...) - if err != nil && cpuLimit != "" && cpuLimit != "0" { - cpuQuantity, err := resource.ParseQuantity(cpuLimit) - if err != nil { - return job, fmt.Errorf("Error parsing cpu limit: %w", err) - } - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity - } else { - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = resource.MustParse("500m") - } - nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...) - if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" { - nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit) - if err != nil { - return job, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err) - } - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity - } - memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...) - if err != nil && memoryLimit != "" && memoryLimit != "0" { - memoryQuantity, err := resource.ParseQuantity(memoryLimit) - if err != nil { - return job, fmt.Errorf("Error parsing memory limit: %w", err) - } - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity - } else { - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = resource.MustParse("512Mi") - } - - cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...) - if err != nil && cpuRequest != "" && cpuRequest != "0" { - cpuQuantity, err := resource.ParseQuantity(cpuRequest) - if err != nil { - return job, fmt.Errorf("Error parsing cpu request: %w", err) - } - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity - } else { - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = resource.MustParse("500m") - } - memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...) - if err != nil && memoryRequest != "" && memoryRequest != "0" { - memoryQuantity, err := resource.ParseQuantity(memoryRequest) - if err != nil { - return job, fmt.Errorf("Error parsing memory request: %w", err) - } - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity - } else { - job.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = resource.MustParse("512Mi") - } - - return job, nil -} - -type Deployment struct { - AppName string - Command []string - Image string - ImagePullSecrets string - ImageSourceType string - Healthchecks []appjson.Healthcheck - Namespace string - PrimaryPort int32 - PortMaps []PortMap - ProcessType string - Replicas int32 - WorkingDir string -} - -func templateKubernetesDeployment(input Deployment) (appsv1.Deployment, error) { - labels := map[string]string{ - "app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), - "app.kubernetes.io/name": input.ProcessType, - "app.kubernetes.io/part-of": input.AppName, - } - annotations := map[string]string{ - "app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED", - "dokku.com/builder-type": input.ImageSourceType, - "dokku.com/managed": "true", - } - secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName) - - podAnnotations := annotations - podAnnotations["kubectl.kubernetes.io/default-container"] = fmt.Sprintf("%s-%s", input.AppName, input.ProcessType) - - deployment := appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), - Namespace: input.Namespace, - Labels: labels, - Annotations: annotations, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: ptr.To(input.Replicas), - RevisionHistoryLimit: ptr.To(int32(5)), - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: podAnnotations, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), - Env: []corev1.EnvVar{}, - EnvFrom: []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, - }, - Optional: ptr.To(true), - }, - }, - }, - Image: input.Image, - ImagePullPolicy: corev1.PullAlways, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{}, - Requests: corev1.ResourceList{}, - }, - WorkingDir: input.WorkingDir, - }, - }, - }, - }, - }, - } - - if len(input.Command) > 0 { - deployment.Spec.Template.Spec.Containers[0].Args = input.Command - } - - if len(input.Healthchecks) > 0 { - livenessChecks := []corev1.Probe{} - readinessChecks := []corev1.Probe{} - startupChecks := []corev1.Probe{} - uptimeSeconds := []int32{} - for _, healthcheck := range input.Healthchecks { - probe := corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{}, - InitialDelaySeconds: healthcheck.InitialDelay, - PeriodSeconds: healthcheck.Wait, - TimeoutSeconds: healthcheck.Timeout, - FailureThreshold: healthcheck.Attempts, - SuccessThreshold: int32(1), - } - if len(healthcheck.Command) > 0 { - probe.ProbeHandler.Exec = &corev1.ExecAction{ - Command: healthcheck.Command, - } - } else if healthcheck.Listening { - probe.ProbeHandler.TCPSocket = &corev1.TCPSocketAction{ - Port: intstr.FromInt32(input.PrimaryPort), - } - for _, header := range healthcheck.HTTPHeaders { - if header.Name == "Host" { - probe.ProbeHandler.TCPSocket.Host = header.Value - } - } - } else if healthcheck.Path != "" { - probe.ProbeHandler.HTTPGet = &corev1.HTTPGetAction{ - Path: healthcheck.Path, - Port: intstr.FromInt32(input.PrimaryPort), - HTTPHeaders: []corev1.HTTPHeader{}, - } - - if healthcheck.Scheme != "" { - probe.ProbeHandler.HTTPGet.Scheme = corev1.URIScheme(strings.ToUpper(healthcheck.Scheme)) - } - - for _, header := range healthcheck.HTTPHeaders { - probe.ProbeHandler.HTTPGet.HTTPHeaders = append(probe.ProbeHandler.HTTPGet.HTTPHeaders, corev1.HTTPHeader{ - Name: header.Name, - Value: header.Value, - }) - } - } else if healthcheck.Uptime > 0 { - uptimeSeconds = append(uptimeSeconds, healthcheck.Uptime) - } - - if healthcheck.Type == appjson.HealthcheckType_Liveness { - livenessChecks = append(livenessChecks, probe) - } else if healthcheck.Type == appjson.HealthcheckType_Readiness { - readinessChecks = append(readinessChecks, probe) - } else if healthcheck.Type == appjson.HealthcheckType_Startup { - startupChecks = append(startupChecks, probe) - } - } - if len(livenessChecks) > 1 { - common.LogWarn("Multiple liveness checks are not supported, only the first one will be used") - } - if len(readinessChecks) > 1 { - common.LogWarn("Multiple readiness checks are not supported, only the first one will be used") - } - if len(startupChecks) > 1 { - common.LogWarn("Multiple startup checks are not supported, only the first one will be used") - } - if len(uptimeSeconds) > 1 { - common.LogWarn("Multiple uptime checks are not supported, only the first one will be used") - } - - if len(livenessChecks) > 0 { - deployment.Spec.Template.Spec.Containers[0].LivenessProbe = &livenessChecks[0] - } - if len(readinessChecks) > 0 { - deployment.Spec.Template.Spec.Containers[0].ReadinessProbe = &readinessChecks[0] - } - if len(startupChecks) > 0 { - deployment.Spec.Template.Spec.Containers[0].StartupProbe = &startupChecks[0] - } - if len(uptimeSeconds) > 0 { - deployment.Spec.MinReadySeconds = uptimeSeconds[0] - } - } - - if input.ProcessType == "web" { - for _, portMap := range input.PortMaps { - protocol := "TCP" - if portMap.Scheme == "udp" { - protocol = "UDP" - } - deployment.Spec.Template.Spec.Containers[0].Ports = append(deployment.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{ - Name: portMap.String(), - ContainerPort: portMap.ContainerPort, - Protocol: corev1.Protocol(protocol), - }) - } - - deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ - Name: "PORT", - Value: fmt.Sprint(input.PrimaryPort), - }) - } - - if input.ImagePullSecrets != "" { - deployment.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{ - { - Name: input.ImagePullSecrets, - }, - } - } - - cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "cpu"}...) - if err != nil && cpuLimit != "" && cpuLimit != "0" { - cpuQuantity, err := resource.ParseQuantity(cpuLimit) - if err != nil { - return deployment, fmt.Errorf("Error parsing cpu limit: %w", err) - } - deployment.Spec.Template.Spec.Containers[0].Resources.Limits["cpu"] = cpuQuantity - } - nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "nvidia-gpu"}...) - if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" { - nvidiaGpuQuantity, err := resource.ParseQuantity(nvidiaGpuLimit) - if err != nil { - return deployment, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err) - } - deployment.Spec.Template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = nvidiaGpuQuantity - } - memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "limit", "memory"}...) - if err != nil && memoryLimit != "" && memoryLimit != "0" { - memoryQuantity, err := resource.ParseQuantity(memoryLimit) - if err != nil { - return deployment, fmt.Errorf("Error parsing memory limit: %w", err) - } - deployment.Spec.Template.Spec.Containers[0].Resources.Limits["memory"] = memoryQuantity - } - - cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "cpu"}...) - if err != nil && cpuRequest != "" && cpuRequest != "0" { - cpuQuantity, err := resource.ParseQuantity(cpuRequest) - if err != nil { - return deployment, fmt.Errorf("Error parsing cpu request: %w", err) - } - deployment.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpuQuantity - } - memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{input.AppName, input.ProcessType, "reserve", "memory"}...) - if err != nil && memoryRequest != "" && memoryRequest != "0" { - memoryQuantity, err := resource.ParseQuantity(memoryRequest) - if err != nil { - return deployment, fmt.Errorf("Error parsing memory request: %w", err) - } - deployment.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memoryQuantity - } - - return deployment, nil -} - -type IngressRouteEntrypoint string - -const ( - IngressRouteEntrypoint_HTTP IngressRouteEntrypoint = "web" - IngressRouteEntrypoint_HTTPS IngressRouteEntrypoint = "websecure" -) - -type IngressRoute struct { - AppName string - Entrypoints []IngressRouteEntrypoint - Namespace string - PortMap PortMap - ProcessType string - ServiceName string -} - -func templateKubernetesIngressRoute(input IngressRoute) traefikv1alpha1.IngressRoute { - labels := map[string]string{ - "app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), - "app.kubernetes.io/name": input.ProcessType, - "app.kubernetes.io/part-of": input.AppName, - } - annotations := map[string]string{ - "dokku.com/managed": "true", - } - - port := input.PortMap.String() - ingressRoute := traefikv1alpha1.IngressRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", input.ServiceName, port), - Namespace: input.Namespace, - Labels: labels, - Annotations: annotations, - }, - Spec: traefikv1alpha1.IngressRouteSpec{ - EntryPoints: []string{string(IngressRouteEntrypoint_HTTP)}, - }, - } - - return ingressRoute -} - func templateKubernetesJob(input Job) (batchv1.Job, error) { labels := map[string]string{ "app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), @@ -907,137 +396,6 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { return job, nil } -type Secret struct { - AppName string - Env map[string]string - Namespace string -} - -func templateKubernetesSecret(input Secret) corev1.Secret { - secretName := fmt.Sprintf("env-%s.DEPLOYMENT_ID", input.AppName) - labels := map[string]string{ - "app.kubernetes.io/instance": secretName, - "app.kubernetes.io/name": fmt.Sprintf("%s-env", input.AppName), - "app.kubernetes.io/part-of": input.AppName, - } - - annotations := map[string]string{ - "app.kubernetes.io/version": "DEPLOYMENT_ID_QUOTED", - "dokku.com/managed": "true", - } - secret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: input.Namespace, - Labels: labels, - Annotations: annotations, - }, - Data: map[string][]byte{}, - } - - return secret -} - -type Service struct { - AppName string - Namespace string - PortMaps []PortMap -} - -func templateKubernetesService(input Service) corev1.Service { - labels := map[string]string{ - "app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, "web"), - "app.kubernetes.io/name": "web", - "app.kubernetes.io/part-of": input.AppName, - } - annotations := map[string]string{ - "dokku.com/managed": "true", - } - service := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", input.AppName, "web"), - Namespace: input.Namespace, - Labels: labels, - Annotations: annotations, - }, - Spec: corev1.ServiceSpec{ - Selector: labels, - }, - } - - for _, portMap := range input.PortMaps { - protocol := "TCP" - if portMap.Scheme == "udp" { - protocol = "UDP" - } - service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{ - Name: portMap.String(), - Port: portMap.HostPort, - TargetPort: intstr.FromString(portMap.String()), - Protocol: corev1.Protocol(protocol), - }) - } - - return service -} - -type WriteResourceInput struct { - AppendContents string - Object runtime.Object - Path string - Replacements *orderedmap.OrderedMap[string, string] -} - -func writeResourceToFile(input WriteResourceInput) error { - common.LogDebug(fmt.Sprintf("Printing resource: %s", input.Path)) - printr := printers.NewTypeSetter(runtimeScheme).ToPrinter(&printers.YAMLPrinter{}) - handle, err := os.Create(input.Path) - if err != nil { - return fmt.Errorf("Error creating template file: %w", err) - } - defer handle.Close() - - if err := printr.PrintObj(input.Object, handle); err != nil { - return fmt.Errorf("Error writing template file: %w", err) - } - - if input.Replacements != nil { - b, err := os.ReadFile(input.Path) - if err != nil { - return fmt.Errorf("Error reading template file: %w", err) - } - - contents := string(b) - for pair := input.Replacements.Oldest(); pair != nil; pair = pair.Next() { - contents = strings.ReplaceAll(string(contents), pair.Key, pair.Value) - } - - err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644)) - if err != nil { - return fmt.Errorf("Error updating template file with replacements: %w", err) - } - } - - if input.AppendContents != "" { - b, err := os.ReadFile(input.Path) - if err != nil { - return fmt.Errorf("Error reading template file: %w", err) - } - - contents := string(b) + "\n" + input.AppendContents - - err = os.WriteFile(input.Path, []byte(contents), os.FileMode(0644)) - if err != nil { - return fmt.Errorf("Error updating template file with replacements: %w", err) - } - } - - if os.Getenv("DOKKU_TRACE") == "1" { - common.CatFile(input.Path) - } - return nil -} - type WriteYamlInput struct { Object interface{} Path string diff --git a/plugins/scheduler-k3s/templates/certificate-append.yaml b/plugins/scheduler-k3s/templates/certificate-append.yaml deleted file mode 100644 index 538ddfdaa..000000000 --- a/plugins/scheduler-k3s/templates/certificate-append.yaml +++ /dev/null @@ -1,4 +0,0 @@ - dnsNames: - {{- range .Values.processes.PROCESS_TYPE.domains }} - - {{ . }} - {{- end }} \ No newline at end of file diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml new file mode 100644 index 000000000..30c385042 --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -0,0 +1,32 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := .Values.processes.PROCESS_NAME }} +{{- if and $config.web.tls $config.domains }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + annotations: + dokku.com/managed: "true" + creationTimestamp: null + labels: + app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: {{ $.Values.global.app_name }}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} +spec: + issuerRef: + kind: ClusterIssuer + name: {{ $config.tls.issuer_name }} + secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }} + secretTemplate: + annotations: + dokku.com/managed: "true" + labels: + app.kubernetes.io/name: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} +status: {} + dnsNames: + {{- range $config.domains }} + - {{ . }} + {{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml new file mode 100644 index 000000000..13fbf4109 --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -0,0 +1,103 @@ +{{- $processName := "CRON_ID" }} +{{- $config := .Values.processes.CRON_ID }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} + dokku.com/builder-type: {{ $.Values.global.image.type }} + dokku.com/cron-id: {{ $config.cron.id }} + dokku.com/job-suffix: {{ $config.cron.suffix }} + dokku.com/managed: "true" + kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} + app.kubernetes.io/name: cron + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + dokku.com/cron-id: {{ $config.cron.id }} + name: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} + namespace: default +spec: + concurrencyPolicy: Allow + failedJobsHistoryLimit: 10 + jobTemplate: + metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} + dokku.com/builder-type: {{ $.Values.global.image.type }} + dokku.com/cron-id: {{ $config.cron.id }} + dokku.com/job-suffix: {{ $config.cron.suffix }} + dokku.com/managed: "true" + kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + creationTimestamp: null + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} + app.kubernetes.io/name: cron + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + dokku.com/cron-id: {{ $config.cron.id }} + spec: + backoffLimit: 0 + podReplacementPolicy: Failed + template: + metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} + dokku.com/builder-type: {{ $.Values.global.image.type }} + dokku.com/cron-id: {{ $config.cron.id }} + dokku.com/job-suffix: {{ $config.cron.suffix }} + dokku.com/managed: "true" + kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-cron + creationTimestamp: null + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-cron-{{ $config.cron.suffix }} + app.kubernetes.io/name: cron + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + dokku.com/cron-id: {{ $config.cron.id }} + spec: + containers: + - args: + {{- range $config.args }} + - {{ . }} + {{- end }} + envFrom: + - secretRef: + name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} + optional: true + image: {{ $.Values.global.image.name }} + imagePullPolicy: Always + name: {{ $.Values.global.app_name }}-cron + {{- if and $config.resources (or $config.resources.limits $config.resources.requests) }} + resources: + {{- if $config.resources.limits }} + limits: + {{- if $config.resources.limits.cpu }} + cpu: {{ $config.resources.limits.cpu }} + {{- end }} + {{- if $config.resources.limits.memory }} + memory: {{ $config.resources.limits.memory }} + {{- end }} + {{- end }} + {{- if $config.resources.requests }} + requests: + {{- if $config.resources.requests.cpu }} + cpu: {{ $config.resources.requests.cpu }} + {{- end }} + {{- if $config.resources.requests.memory }} + memory: {{ $config.resources.requests.memory }} + {{- end }} + {{- end }} + {{- end }} + {{- if $.Values.global.image.working_dir }} + workingDir: {{ $.Values.global.image.working_dir }} + {{- end }} + {{- if $.Values.global.image.image_pull_secrets }} + imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }} + {{- end }} + restartPolicy: Never + ttlSecondsAfterFinished: 60 + schedule: {{ $config.cron.schedule }} + startingDeadlineSeconds: 60 + successfulJobsHistoryLimit: 10 + suspend: false + timeZone: Etc/UTC \ No newline at end of file diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml new file mode 100644 index 000000000..89c6fbb7e --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -0,0 +1,96 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := .Values.processes.PROCESS_NAME }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} + dokku.com/builder-type: {{ $.Values.global.image.type }} + dokku.com/managed: "true" + kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }} + creationTimestamp: null + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: {{ $.Values.global.app_name }}-{{ $processName }} + namespace: default +spec: + replicas: {{ $config.replicas }} + revisionHistoryLimit: 5 + selector: + matchLabels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + strategy: {} + template: + metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} + dokku.com/builder-type: {{ $.Values.global.image.type }} + dokku.com/managed: "true" + kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }} + creationTimestamp: null + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + spec: + containers: + - args: + {{- range $config.args }} + - {{ . }} + {{- end }} + {{- if eq $processName "web" }} + env: + - name: PORT + value: "{{ $.Values.global.primary_port }}" + {{- end }} + envFrom: + - secretRef: + name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} + optional: true + image: {{ $.Values.global.image.name }} + imagePullPolicy: Always + name: {{ $.Values.global.app_name }}-{{ $processName }} + {{- if eq $processName "web" }} + ports: + {{- range $pdx, $port_map := $config.web.port_maps }} + - containerPort: {{ $port_map.container_port }} + name: {{ $port_map.name }} + protocol: {{ $port_map.protocol }} + {{- end }} + {{- end }} + {{- if and $config.resources (or $config.resources.limits $config.resources.requests) }} + resources: + {{- if $config.resources.limits }} + limits: + {{- if $config.resources.limits.cpu }} + cpu: {{ $config.resources.limits.cpu }} + {{- end }} + {{- if $config.resources.limits.memory }} + memory: {{ $config.resources.limits.memory }} + {{- end }} + {{- end }} + {{- if $config.resources.requests }} + requests: + {{- if $config.resources.requests.cpu }} + cpu: {{ $config.resources.requests.cpu }} + {{- end }} + {{- if $config.resources.requests.memory }} + memory: {{ $config.resources.requests.memory }} + {{- end }} + {{- end }} + {{- end }} + {{- if $config.healthchecks.startup }} + startupProbe: + {{ $config.healthchecks.startup | toJson | indent 10 }} + {{- end }} + {{- if $.Values.global.image.working_dir }} + workingDir: {{ $.Values.global.image.working_dir }} + {{- end }} + {{- if $.Values.global.image.image_pull_secrets }} + imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }} + {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml new file mode 100644 index 000000000..444a25c88 --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -0,0 +1,35 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := .Values.processes.PROCESS_NAME }} +{{- if $config.web.domains }} +{{- range $pdx, $port_map := $config.web.port_maps }} +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + annotations: + dokku.com/managed: "true" + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }} + namespace: {{ $.Values.global.namespace }} +spec: + entryPoints: + - web + {{- if and $config.web.tls (eq $port_map "https") }} + - websecure + {{- end }} + routes: + {{- range $ddx, $domain := $config.web.domains }} + - kind: Rule + match: Host(`{{ $domain }}`) + services: + - name: {{ $.Values.global.app_name }}-{{ $processName }} + namespace: {{ $.Values.global.namespace }} + passHostHeader: true + port: {{ $port_map.name }} + scheme: {{ $port_map.scheme }} + {{- end }} +{{- end }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/secret.yaml b/plugins/scheduler-k3s/templates/chart/secret.yaml new file mode 100644 index 000000000..a0115ac2b --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/secret.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} + dokku.com/managed: "true" + labels: + app.kubernetes.io/instance: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} + app.kubernetes.io/name: env-{{ $.Values.global.app_name }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} + namespace: {{ $.Values.global.namespace }} +{{- with .Values.global.secrets }} +data: + {{- toYaml . | nindent 2 }} +{{- end }} \ No newline at end of file diff --git a/plugins/scheduler-k3s/templates/chart/service.yaml b/plugins/scheduler-k3s/templates/chart/service.yaml new file mode 100644 index 000000000..a3ed18c83 --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/service.yaml @@ -0,0 +1,26 @@ +{{- $processName := "PROCESS_NAME" }} +{{- $config := .Values.processes.PROCESS_NAME }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + dokku.com/managed: "true" + labels: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: {{ $.Values.global.app_name }}-{{ $processName }} + namespace: default +spec: + ports: + {{- range $pdx, $port_map := $config.web.port_maps }} + - name: {{ $port_map.name }} + port: {{ $port_map.host_port }} + protocol: TCP + targetPort: {{ $port_map.name }} + {{- end }} + selector: + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} + app.kubernetes.io/name: {{ $processName }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} diff --git a/plugins/scheduler-k3s/templates/cert-manager.yaml b/plugins/scheduler-k3s/templates/helm-config/cert-manager.yaml similarity index 100% rename from plugins/scheduler-k3s/templates/cert-manager.yaml rename to plugins/scheduler-k3s/templates/helm-config/cert-manager.yaml diff --git a/plugins/scheduler-k3s/templates/longhorn.yaml b/plugins/scheduler-k3s/templates/helm-config/longhorn.yaml similarity index 100% rename from plugins/scheduler-k3s/templates/longhorn.yaml rename to plugins/scheduler-k3s/templates/helm-config/longhorn.yaml diff --git a/plugins/scheduler-k3s/templates/traefik-config.yaml b/plugins/scheduler-k3s/templates/helm-config/traefik-config.yaml similarity index 100% rename from plugins/scheduler-k3s/templates/traefik-config.yaml rename to plugins/scheduler-k3s/templates/helm-config/traefik-config.yaml diff --git a/plugins/scheduler-k3s/templates/ingress-routes-append.yaml b/plugins/scheduler-k3s/templates/ingress-routes-append.yaml deleted file mode 100644 index 237ebeef9..000000000 --- a/plugins/scheduler-k3s/templates/ingress-routes-append.yaml +++ /dev/null @@ -1,18 +0,0 @@ - {{ if .Values.processes.PROCESS_TYPE.tls }} - - websecure - {{- end }} - routes: - {{- range .Values.processes.PROCESS_TYPE.domains }} - - kind: Rule - match: Host(`{{ . }}`) - services: - - name: APP_NAME-PROCESS_TYPE - namespace: NAMESPACE - passHostHeader: true - port: PORT_MAPPING - scheme: PORT_SCHEME - {{- end }} - {{ if .Values.processes.PROCESS_TYPE.tls }} - tls: - secretName: tls-APP_NAME-PROCESS_TYPE - {{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index fea7c18ac..e5ec1492c 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -3,6 +3,7 @@ package scheduler_k3s import ( "bufio" "context" + "crypto/rand" "encoding/base64" "encoding/json" "errors" @@ -26,9 +27,7 @@ import ( "github.com/kballard/go-shellquote" "github.com/rancher/wharfie/pkg/registries" "github.com/ryanuber/columnize" - orderedmap "github.com/wk8/go-ordered-map/v2" "gopkg.in/yaml.v3" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/client/conditions" @@ -227,34 +226,26 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e if err != nil { return fmt.Errorf("Error creating chart directory: %w", err) } - defer os.RemoveAll(chartDir) + // defer os.RemoveAll(chartDir) if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil { return fmt.Errorf("Error creating chart templates directory: %w", err) } deploymentId := time.Now().Unix() - replacements := orderedmap.New[string, string]() - replacements.Set("DEPLOYMENT_ID_QUOTED", "{{.Values.deploment_id | quote}}") - replacements.Set("DEPLOYMENT_ID", "{{.Values.deploment_id}}") - - secret := templateKubernetesSecret(Secret{ - AppName: appName, - Env: env.Map(), - Namespace: namespace, - }) - err = writeResourceToFile(WriteResourceInput{ - Object: &secret, - Path: filepath.Join(chartDir, "templates/secret.yaml"), - Replacements: replacements, - AppendContents: `{{- with .Values.secrets }} -data: - {{- toYaml . | nindent 2 }} -{{- end }} -`, - }) + b, err := templates.ReadFile("templates/chart/secret.yaml") if err != nil { - return fmt.Errorf("Error printing deployment: %w", err) + return fmt.Errorf("Error reading secret template: %w", err) + } + + filename := filepath.Join(chartDir, "templates", "secrets.yaml") + err = os.WriteFile(filename, b, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("Error writing secrets template: %w", err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(filename) } portMaps, err := getPortMaps(appName) @@ -276,125 +267,12 @@ data: } workingDir := common.GetWorkingDir(appName, image) - deployments := map[string]appsv1.Deployment{} - i := 0 - for processType := range processes { - startCommand, err := getStartCommand(StartCommandInput{ - AppName: appName, - ProcessType: processType, - ImageSourceType: imageSourceType, - Port: primaryPort, - Env: env.Map(), - }) - if err != nil { - return fmt.Errorf("Error getting start command for deployment: %w", err) - } - - i++ - replicaCountPlaceholder := int32(i * 1000) - - healthchecks, ok := appJSON.Healthchecks[processType] - if !ok { - healthchecks = []appjson.Healthcheck{} - } - - // todo: implement deployment annotations - // todo: implement pod annotations - // todo: implement volumes - deployment, err := templateKubernetesDeployment(Deployment{ - AppName: appName, - Command: startCommand.Command, - Image: image, - ImagePullSecrets: imagePullSecrets, - ImageSourceType: imageSourceType, - Healthchecks: healthchecks, - Namespace: namespace, - PrimaryPort: primaryPort, - PortMaps: portMaps, - ProcessType: processType, - Replicas: replicaCountPlaceholder, - WorkingDir: workingDir, - }) - if err != nil { - return fmt.Errorf("Error templating deployment: %w", err) - } - - replacements.Set(fmt.Sprintf("replicas: %d", replicaCountPlaceholder), fmt.Sprintf("replicas: {{.Values.processes.%s.replicas}}", processType)) - deployments[processType] = deployment - err = writeResourceToFile(WriteResourceInput{ - Object: &deployment, - Path: filepath.Join(chartDir, fmt.Sprintf("templates/deployment-%s.yaml", deployment.Name)), - Replacements: replacements, - }) - if err != nil { - return fmt.Errorf("Error printing deployment: %w", err) - } - - replacements.Delete(fmt.Sprintf("replicas: %d", replicaCountPlaceholder)) - } cronEntries, err := cron.FetchCronEntries(appName) if err != nil { return fmt.Errorf("Error fetching cron entries: %w", err) } - clientset, err := NewKubernetesClient() - if err != nil { - return fmt.Errorf("Error creating kubernetes client: %w", err) - } - - cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{ - LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), - Namespace: namespace, - }) - if err != nil { - return fmt.Errorf("Error listing cron jobs: %w", err) - } - - for _, cronEntry := range cronEntries { - suffix := "" - for _, cronJob := range cronJobs { - if cronJob.Labels["dokku.com/cron-id"] == cronEntry.ID { - var ok bool - suffix, ok = cronJob.Annotations["dokku.com/job-suffix"] - if !ok { - suffix = "" - } - } - } - - words, err := shellquote.Split(cronEntry.Command) - if err != nil { - return fmt.Errorf("Error parsing cron command: %w", err) - } - cronJob, err := templateKubernetesCronJob(Job{ - AppName: appName, - Command: words, - Env: map[string]string{}, - ID: cronEntry.ID, - Image: image, - ImagePullSecrets: imagePullSecrets, - ImageSourceType: imageSourceType, - Namespace: namespace, - ProcessType: "cron", - Schedule: cronEntry.Schedule, - Suffix: suffix, - WorkingDir: workingDir, - }) - if err != nil { - return fmt.Errorf("Error templating cron job: %w", err) - } - - err = writeResourceToFile(WriteResourceInput{ - Object: &cronJob, - Path: filepath.Join(chartDir, fmt.Sprintf("templates/cron-job-%s.yaml", cronEntry.ID)), - Replacements: replacements, - }) - if err != nil { - return fmt.Errorf("Error printing cron job: %w", err) - } - } - issuerName := "letsencrypt-stag" server := getComputedLetsencryptServer(appName) if server == "prod" || server == "production" { @@ -412,22 +290,7 @@ data: } domains := []string{} - if deployment, ok := deployments["web"]; ok { - service := templateKubernetesService(Service{ - AppName: appName, - Namespace: namespace, - PortMaps: portMaps, - }) - - err := writeResourceToFile(WriteResourceInput{ - Object: &service, - Path: filepath.Join(chartDir, "templates/service-web.yaml"), - Replacements: replacements, - }) - if err != nil { - return fmt.Errorf("Error printing service: %w", err) - } - + if _, ok := processes["web"]; ok { err = common.PlugnTrigger("domains-vhost-enabled", []string{appName}...) if err == nil { b, err := common.PlugnTriggerOutput("domains-list", []string{appName}...) @@ -442,34 +305,6 @@ data: } } } - - err = createIngressRoutesFiles(CreateIngressRoutesInput{ - AppName: appName, - ChartDir: chartDir, - Deployment: deployment, - Namespace: namespace, - PortMaps: portMaps, - ProcessType: "web", - Service: service, - }) - if err != nil { - return fmt.Errorf("Error creating ingress routes: %w", err) - } - - err = createCertificateFile(CreateCertificateFileInput{ - ChartDir: chartDir, - Certificate: Certificate{ - AppName: appName, - Name: fmt.Sprintf("%s-%s", appName, "web"), - Namespace: namespace, - TLS: tls, - }, - IssuerName: issuerName, - ProcessType: "web", - }) - if err != nil { - return fmt.Errorf("Error creating certificate files: %w", err) - } } chart := &Chart{ @@ -488,25 +323,181 @@ data: } values := &Values{ - DeploymentID: fmt.Sprint(deploymentId), - Secrets: map[string]string{}, - Processes: map[string]ProcessValues{}, + Global: GlobalValues{ + AppName: appName, + DeploymentID: fmt.Sprint(deploymentId), + Image: GlobalImage{ + ImagePullSecrets: imagePullSecrets, + Name: image, + Type: imageSourceType, + WorkingDir: workingDir, + }, + Namespace: namespace, + PrimaryPort: primaryPort, + Secrets: map[string]string{}, + }, + Processes: map[string]ProcessValues{}, } + for processType, processCount := range processes { + // todo: implement deployment annotations + // todo: implement pod annotations + // todo: implement volumes + + healthchecks, ok := appJSON.Healthchecks[processType] + if !ok { + healthchecks = []appjson.Healthcheck{} + } + processHealthchecks := getProcessHealtchecks(healthchecks, primaryPort) + + startCommand, err := getStartCommand(StartCommandInput{ + AppName: appName, + ProcessType: processType, + ImageSourceType: imageSourceType, + Port: primaryPort, + Env: env.Map(), + }) + if err != nil { + return fmt.Errorf("Error getting start command for deployment: %w", err) + } + args := startCommand.Command + + processResources, err := getProcessResources(appName, processType) + if err != nil { + return fmt.Errorf("Error getting process resources: %w", err) + } + processValues := ProcessValues{ - Replicas: int32(processCount), + Args: args, + Healthchecks: processHealthchecks, + ProcessType: ProcessType_Worker, + Replicas: int32(processCount), + Resources: processResources, } if processType == "web" { - sort.Strings(domains) - processValues.Domains = domains - processValues.TLS = tls + processValues.Web = ProcessWeb{ + Domains: domains, + PortMaps: []ProcessPortMap{}, + TLS: tls, + } + processValues.ProcessType = ProcessType_Web + for _, portMap := range portMaps { + protocol := PortmapProtocol_TCP + if portMap.Scheme == "udp" { + protocol = PortmapProtocol_UDP + } + processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{ + ContainerPort: portMap.ContainerPort, + HostPort: portMap.HostPort, + Name: portMap.String(), + Protocol: protocol, + Scheme: portMap.Scheme, + }) + } + + sort.Sort(NameSorter(processValues.Web.PortMaps)) + sort.Strings(processValues.Web.Domains) } values.Processes[processType] = processValues + + for _, templateName := range []string{"deployment", "service", "certificate", "ingress-route"} { + b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) + if err != nil { + return fmt.Errorf("Error reading %s template: %w", templateName, err) + } + + filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s-%s.yaml", templateName, processType)) + contents := strings.ReplaceAll(string(b), "PROCESS_NAME", processType) + err = os.WriteFile(filename, []byte(contents), os.FileMode(0644)) + if err != nil { + return fmt.Errorf("Error writing %s template: %w", templateName, err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(filename) + } + } + + } + + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Error creating kubernetes client: %w", err) + } + + cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{ + LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), + Namespace: namespace, + }) + if err != nil { + return fmt.Errorf("Error listing cron jobs: %w", err) + } + for _, cronEntry := range cronEntries { + // todo: implement deployment annotations + // todo: implement pod annotations + // todo: implement volumes + suffix := "" + for _, cronJob := range cronJobs { + if cronJob.Labels["dokku.com/cron-id"] == cronEntry.ID { + var ok bool + suffix, ok = cronJob.Annotations["dokku.com/job-suffix"] + if !ok { + suffix = "" + } + } + } + if suffix == "" { + n := 5 + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + panic(err) + } + suffix = strings.ToLower(fmt.Sprintf("%X", b)) + } + + words, err := shellquote.Split(cronEntry.Command) + if err != nil { + return fmt.Errorf("Error parsing cron command: %w", err) + } + + processResources, err := getProcessResources(appName, cronEntry.ID) + if err != nil { + return fmt.Errorf("Error getting process resources: %w", err) + } + + processValues := ProcessValues{ + Args: words, + Cron: ProcessCron{ + ID: cronEntry.ID, + Schedule: cronEntry.Schedule, + Suffix: suffix, + }, + ProcessType: ProcessType_Cron, + Replicas: 1, + Resources: processResources, + } + values.Processes[cronEntry.ID] = processValues + + b, err := templates.ReadFile("templates/chart/cron-job.yaml") + if err != nil { + return fmt.Errorf("Error reading cron job template: %w", err) + } + + cronFile := filepath.Join(chartDir, "templates", fmt.Sprintf("cron-job-%s.yaml", cronEntry.ID)) + contents := strings.ReplaceAll(string(b), "CRON_ID", cronEntry.ID) + err = os.WriteFile(cronFile, []byte(contents), os.FileMode(0644)) + if err != nil { + return fmt.Errorf("Error writing cron job template: %w", err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(cronFile) + } } for key, value := range env.Map() { - values.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value)) + values.Global.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value)) } err = writeYaml(WriteYamlInput{