diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index 62348ff5a..52a7281ac 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -13,6 +13,7 @@ require ( github.com/go-openapi/jsonpointer v0.20.2 github.com/go-resty/resty/v2 v2.11.0 github.com/gofrs/flock v0.8.1 + github.com/gosimple/slug v1.14.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kedacore/keda/v2 v2.13.0 github.com/ryanuber/columnize v2.1.2+incompatible @@ -86,6 +87,7 @@ require ( github.com/google/uuid v1.5.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/gosimple/unidecode v1.0.1 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum index e613ee475..0fedfa358 100644 --- a/plugins/scheduler-k3s/go.sum +++ b/plugins/scheduler-k3s/go.sum @@ -195,6 +195,10 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gosimple/slug v1.14.0 h1:RtTL/71mJNDfpUbCOmnf/XFkzKRtD6wL6Uy+3akm4Es= +github.com/gosimple/slug v1.14.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= +github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= +github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index b0cc95fcb..098bf3ad1 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -1,6 +1,7 @@ package scheduler_k3s import ( + "bytes" "context" "errors" "fmt" @@ -291,6 +292,16 @@ func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Re return releases, nil } +type DebugRenderer struct { +} + +func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { + for _, line := range strings.Split(renderedManifests.String(), "\n") { + common.LogWarn(line) + } + return renderedManifests, nil +} + func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { namespace := input.Namespace if namespace == "" { @@ -311,8 +322,11 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error { client.Atomic = input.RollbackOnFailure client.ChartPathOptions = action.ChartPathOptions{} client.CleanupOnFail = true - client.DryRun = false client.MaxHistory = 10 + if os.Getenv("DOKKU_TRACE") == "1" { + client.DryRun = true + client.PostRenderer = &DebugRenderer{} + } client.Namespace = namespace client.Timeout = input.Timeout client.Wait = input.Wait diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index f7f13245e..6cc5835f5 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -12,6 +12,7 @@ import ( autoscalingv1 "k8s.io/api/autoscaling/v1" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -229,6 +230,20 @@ func (k KubernetesClient) CreateNamespace(ctx context.Context, input CreateNames return *namespace, err } +// DeleteIngressInput contains all the information needed to delete a Kubernetes ingress +type DeleteIngressInput struct { + // Name is the Kubernetes ingress name + Name string + + // Namespace is the Kubernetes namespace + Namespace string +} + +// DeleteIngress deletes a Kubernetes ingress +func (k KubernetesClient) DeleteIngress(ctx context.Context, input DeleteIngressInput) error { + return k.Client.NetworkingV1().Ingresses(input.Namespace).Delete(ctx, input.Name, metav1.DeleteOptions{}) +} + // DeleteJobInput contains all the information needed to delete a Kubernetes job type DeleteJobInput struct { // Name is the Kubernetes job name @@ -438,6 +453,30 @@ func (k KubernetesClient) ListDeployments(ctx context.Context, input ListDeploym return deployments.Items, nil } +// ListIngressesInput contains all the information needed to list Kubernetes ingresses +type ListIngressesInput struct { + // Namespace is the Kubernetes namespace + Namespace string + + // LabelSelector is the Kubernetes label selector + LabelSelector string +} + +// ListIngresses lists Kubernetes ingresses +func (k KubernetesClient) ListIngresses(ctx context.Context, input ListIngressesInput) ([]networkingv1.Ingress, error) { + listOptions := metav1.ListOptions{LabelSelector: input.LabelSelector} + ingresses, err := k.Client.NetworkingV1().Ingresses(input.Namespace).List(ctx, listOptions) + if err != nil { + return []networkingv1.Ingress{}, err + } + + if ingresses == nil { + return []networkingv1.Ingress{}, errors.New("ingresses is nil") + } + + return ingresses.Items, nil +} + // ListNamespaces lists Kubernetes namespaces func (k KubernetesClient) ListNamespaces(ctx context.Context) ([]v1.Namespace, error) { namespaces, err := k.Client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index d5fb94ba3..6622ba17b 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -238,11 +238,16 @@ type ProcessLabels struct { } type ProcessWeb struct { - Domains []string `yaml:"domains,omitempty"` + Domains []ProcessDomains `yaml:"domains,omitempty"` PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"` TLS ProcessTls `yaml:"tls"` } +type ProcessDomains struct { + Name string `yaml:"name"` + Slug string `yaml:"slug"` +} + type ProcessResourcesMap struct { Limits ProcessResources `yaml:"limits,omitempty"` Requests ProcessResources `yaml:"requests,omitempty"` diff --git a/plugins/scheduler-k3s/templates/chart/_helpers.tpl b/plugins/scheduler-k3s/templates/chart/_helpers.tpl index 3f340c56c..6fb5ee787 100644 --- a/plugins/scheduler-k3s/templates/chart/_helpers.tpl +++ b/plugins/scheduler-k3s/templates/chart/_helpers.tpl @@ -15,3 +15,35 @@ {{- end }} {{- end }} {{- end }} + +{{- define "primary.port" -}} +{{- $found := dict -}} +{{- range $idx, $port_map := . -}} +{{- if hasPrefix "https-443-" $port_map.name -}} +{{- $found := set $found "https-443" $port_map.name -}} +{{- end -}} +{{- if hasPrefix "http-80-" $port_map.name -}} +{{- $found := set $found "http-80" $port_map.name -}} +{{- end -}} +{{- if and (eq "https" $port_map.scheme) (not (hasKey $found "https")) -}} +{{- $found := set $found "https" $port_map.name -}} +{{- end -}} +{{- if and (eq "http" $port_map.scheme) (not (hasKey $found "http")) -}} +{{- $found := set $found "http" $port_map.name -}} +{{- end -}} +{{- if not (hasKey $found "any") -}} +{{- $found := set $found "any" $port_map.name -}} +{{- end -}} +{{- end -}} +{{- if hasKey $found "https-443" -}} +{{- get $found "https-443" -}} +{{- else if hasKey $found "http-80" -}} +{{- get $found "http-80" -}} +{{- else if hasKey $found "https" -}} +{{- get $found "https" -}} +{{- else if hasKey $found "http" -}} +{{- get $found "http" -}} +{{- else if hasKey $found "any" -}} +{{- get $found "any" -}} +{{- end -}} +{{- end -}} diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml index 1ffc42a92..029230020 100644 --- a/plugins/scheduler-k3s/templates/chart/certificate.yaml +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -30,7 +30,7 @@ spec: {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 6 }} {{ include "print.labels" (dict "config" $config "key" "secret") | indent 6 }} dnsNames: - {{- range $config.web.domains }} - - {{ . }} + {{- range $idx, $domain := $config.web.domains }} + - {{ $domain.name }} {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index e61c48063..306e4a7a3 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -34,7 +34,7 @@ spec: routes: {{- range $ddx, $domain := $config.web.domains }} - kind: Rule - match: Host(`{{ $domain }}`) + match: Host(`{{ $domain.name }}`) {{- if $config.web.tls.enabled }} middlewares: - name: {{ $.Values.global.app_name}}-{{ $processName }}-redirect-to-https diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index cfbf519df..b0eaa6c57 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,20 +1,14 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} -{{- $mappings := dict }} -{{- range $pdx, $port_map := $config.web.port_maps }} -{{- $mappings := set $mappings $port_map.name "true" }} -{{- end }} -{{- range $pdx, $port_map := $config.web.port_maps }} -{{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }} -{{- continue }} -{{- end }} +{{- range $pdx, $domain := $config.web.domains }} --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: dokku.com/managed: "true" + dokku.com/ingress-method: "domains" {{- if $config.web.tls.enabled }} nginx.ingress.kubernetes.io/force-ssl-redirect: "true" {{- end }} @@ -26,30 +20,28 @@ metadata: app.kubernetes.io/part-of: {{ $.Values.global.app_name }} {{ include "print.labels" (dict "config" $.Values.global "key" "ingress") | indent 4 }} {{ include "print.labels" (dict "config" $config "key" "ingress") | indent 4 }} - name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }} + name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $domain.slug }} namespace: {{ $.Values.global.namespace }} spec: ingressClassName: nginx {{- if $config.web.tls.enabled }} tls: - hosts: - {{- range $ddx, $domain := $config.web.domains }} - - {{ $domain | quote }} - {{- end }} + - {{ $domain.name | quote }} secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }} {{- end }} rules: - {{- range $ddx, $domain := $config.web.domains }} - - host: {{ $domain | quote }} + - host: {{ $domain.name | quote }} http: paths: + {{- if $config.web.port_maps }} - backend: service: name: {{ $.Values.global.app_name }}-{{ $processName }} port: - name: {{ $port_map.name }} + name: {{ include "primary.port" $config.web.port_maps }} pathType: ImplementationSpecific path: / - {{- end }} + {{- end }} {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 827380869..f0d47120c 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -23,6 +23,7 @@ import ( "github.com/dokku/dokku/plugins/config" "github.com/dokku/dokku/plugins/cron" "github.com/fatih/color" + "github.com/gosimple/slug" "github.com/kballard/go-shellquote" "github.com/ryanuber/columnize" corev1 "k8s.io/api/core/v1" @@ -36,7 +37,6 @@ func TriggerInstall() error { if err := common.PropertySetup("scheduler-k3s"); err != nil { return fmt.Errorf("Unable to install the scheduler-k3s plugin: %s", err.Error()) } - return nil } @@ -385,8 +385,17 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } if processType == "web" { + sort.Strings(domains) + domainValues := []ProcessDomains{} + for _, domain := range domains { + domainValues = append(domainValues, ProcessDomains{ + Name: domain, + Slug: slug.Make(domain), + }) + } + processValues.Web = ProcessWeb{ - Domains: domains, + Domains: domainValues, PortMaps: []ProcessPortMap{}, TLS: ProcessTls{ Enabled: tlsEnabled, @@ -435,7 +444,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } sort.Sort(NameSorter(processValues.Web.PortMaps)) - sort.Strings(processValues.Web.Domains) } values.Processes[processType] = processValues @@ -587,7 +595,39 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error parsing deploy timeout duration: %w", err) } - common.LogExclaim(fmt.Sprintf("Installing %s", appName)) + ingresses, err := clientset.ListIngresses(ctx, ListIngressesInput{ + Namespace: namespace, + LabelSelector: fmt.Sprintf("app.kubernetes.io/instance=%s-web", appName), + }) + if err != nil { + return fmt.Errorf("Error listing ingresses: %w", err) + } + + ingressesToDelete := []string{} + + for _, ingress := range ingresses { + ingressIngressMethod := ingress.Annotations["dokku.com/ingress-method"] + if ingressIngressMethod != "domains" { + ingressesToDelete = append(ingressesToDelete, ingress.Name) + } + } + + if len(ingressesToDelete) > 0 { + common.LogWarn("Manually removing non-matching ingress resources") + } + for _, ingressName := range ingressesToDelete { + common.LogVerboseQuiet(fmt.Sprintf("Removing non-matching ingress resource: %s", ingressName)) + err := clientset.DeleteIngress(ctx, DeleteIngressInput{ + Name: ingressName, + Namespace: namespace, + }) + + if err != nil { + return fmt.Errorf("Error deleting ingress: %w", err) + } + } + + common.LogInfo2(fmt.Sprintf("Installing %s", appName)) err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{ ChartPath: chartPath, Namespace: namespace, diff --git a/tests/unit/scheduler-k3s-1.bats b/tests/unit/scheduler-k3s-1.bats index b8a62e8d8..f9fbcd455 100644 --- a/tests/unit/scheduler-k3s-1.bats +++ b/tests/unit/scheduler-k3s-1.bats @@ -32,3 +32,33 @@ teardown_() { INGRESS_CLASS=nginx TAINT_SCHEDULING=true install_k3s } + +@test "(scheduler-k3s) deploy dockerfile exposed port" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:from-image $TEST_APP metabase/metabase:latest" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + assert_http_localhost_response_contains "http" "$TEST_APP.dokku.me" "80" "" "Metabase" +} diff --git a/tests/unit/test_helper.bash b/tests/unit/test_helper.bash index 831f94aa9..56dba8a87 100644 --- a/tests/unit/test_helper.bash +++ b/tests/unit/test_helper.bash @@ -238,6 +238,22 @@ assert_http_localhost_response() { fi } +assert_http_localhost_response_contains() { + local scheme="$1" domain="$2" port="${3:-80}" path="${4:-}" content="${5:-}" status_code="${6:-200}" + run curl --connect-to "$domain:$port:localhost:$port" -kSso /dev/null -w "%{http_code}" "$scheme://$domain:$port$path" + echo "curl: curl --connect-to $domain:$port:localhost:$port -kSso /dev/null -w %{http_code} $scheme://$domain:$port$path" + echo "output: $output" + echo "status: $status" + assert_output "$status_code" + + if [[ -n "$content" ]]; then + run curl --connect-to "$domain:$port:localhost:$port" -kSs "$scheme://$domain:$port$path" + echo "output: $output" + echo "status: $status" + assert_output_contains "$content" + fi +} + assert_ssl_domain() { local domain=$1 assert_app_domain "${domain}"