Merge pull request #6681 from dokku/k3s-ingress-per-domain

Migrate to single ingress per app/domain combination

plugins/scheduler-k3s/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.20.2
 	github.com/go-resty/resty/v2 v2.11.0
 	github.com/gofrs/flock v0.8.1
+	github.com/gosimple/slug v1.14.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kedacore/keda/v2 v2.13.0
 	github.com/ryanuber/columnize v2.1.2+incompatible
@@ -86,6 +87,7 @@ require (
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect

plugins/scheduler-k3s/go.sum

@@ -195,6 +195,10 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/gosimple/slug v1.14.0 h1:RtTL/71mJNDfpUbCOmnf/XFkzKRtD6wL6Uy+3akm4Es=
+github.com/gosimple/slug v1.14.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
+github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o=
+github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
 github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
 github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=

plugins/scheduler-k3s/helm.go

@@ -1,6 +1,7 @@
 package scheduler_k3s
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -291,6 +292,16 @@ func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Re
 	return releases, nil
 }
 
+type DebugRenderer struct {
+}
+
+func (p *DebugRenderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
+	for _, line := range strings.Split(renderedManifests.String(), "\n") {
+		common.LogWarn(line)
+	}
+	return renderedManifests, nil
+}
+
 func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
 	namespace := input.Namespace
 	if namespace == "" {
@@ -311,8 +322,11 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
 	client.Atomic = input.RollbackOnFailure
 	client.ChartPathOptions = action.ChartPathOptions{}
 	client.CleanupOnFail = true
-	client.DryRun = false
 	client.MaxHistory = 10
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		client.DryRun = true
+		client.PostRenderer = &DebugRenderer{}
+	}
 	client.Namespace = namespace
 	client.Timeout = input.Timeout
 	client.Wait = input.Wait

plugins/scheduler-k3s/k8s.go

@@ -12,6 +12,7 @@ import (
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -229,6 +230,20 @@ func (k KubernetesClient) CreateNamespace(ctx context.Context, input CreateNames
 	return *namespace, err
 }
 
+// DeleteIngressInput contains all the information needed to delete a Kubernetes ingress
+type DeleteIngressInput struct {
+	// Name is the Kubernetes ingress name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+}
+
+// DeleteIngress deletes a Kubernetes ingress
+func (k KubernetesClient) DeleteIngress(ctx context.Context, input DeleteIngressInput) error {
+	return k.Client.NetworkingV1().Ingresses(input.Namespace).Delete(ctx, input.Name, metav1.DeleteOptions{})
+}
+
 // DeleteJobInput contains all the information needed to delete a Kubernetes job
 type DeleteJobInput struct {
 	// Name is the Kubernetes job name
@@ -438,6 +453,30 @@ func (k KubernetesClient) ListDeployments(ctx context.Context, input ListDeploym
 	return deployments.Items, nil
 }
 
+// ListIngressesInput contains all the information needed to list Kubernetes ingresses
+type ListIngressesInput struct {
+	// Namespace is the Kubernetes namespace
+	Namespace string
+
+	// LabelSelector is the Kubernetes label selector
+	LabelSelector string
+}
+
+// ListIngresses lists Kubernetes ingresses
+func (k KubernetesClient) ListIngresses(ctx context.Context, input ListIngressesInput) ([]networkingv1.Ingress, error) {
+	listOptions := metav1.ListOptions{LabelSelector: input.LabelSelector}
+	ingresses, err := k.Client.NetworkingV1().Ingresses(input.Namespace).List(ctx, listOptions)
+	if err != nil {
+		return []networkingv1.Ingress{}, err
+	}
+
+	if ingresses == nil {
+		return []networkingv1.Ingress{}, errors.New("ingresses is nil")
+	}
+
+	return ingresses.Items, nil
+}
+
 // ListNamespaces lists Kubernetes namespaces
 func (k KubernetesClient) ListNamespaces(ctx context.Context) ([]v1.Namespace, error) {
 	namespaces, err := k.Client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})

plugins/scheduler-k3s/template.go

@@ -238,11 +238,16 @@ type ProcessLabels struct {
 }
 
 type ProcessWeb struct {
-	Domains  []string         `yaml:"domains,omitempty"`
+	Domains  []ProcessDomains `yaml:"domains,omitempty"`
 	PortMaps []ProcessPortMap `yaml:"port_maps,omitempty"`
 	TLS      ProcessTls       `yaml:"tls"`
 }
 
+type ProcessDomains struct {
+	Name string `yaml:"name"`
+	Slug string `yaml:"slug"`
+}
+
 type ProcessResourcesMap struct {
 	Limits   ProcessResources `yaml:"limits,omitempty"`
 	Requests ProcessResources `yaml:"requests,omitempty"`

plugins/scheduler-k3s/templates/chart/_helpers.tpl

@@ -15,3 +15,35 @@
 {{- end }}
 {{- end }}
 {{- end }}
+
+{{- define "primary.port" -}}
+{{- $found := dict -}}
+{{- range $idx, $port_map := . -}}
+{{- if hasPrefix "https-443-" $port_map.name -}}
+{{- $found := set $found "https-443" $port_map.name -}}
+{{- end -}}
+{{- if hasPrefix "http-80-" $port_map.name -}}
+{{- $found := set $found "http-80" $port_map.name -}}
+{{- end -}}
+{{- if and (eq "https" $port_map.scheme) (not (hasKey $found "https")) -}}
+{{- $found := set $found "https" $port_map.name -}}
+{{- end -}}
+{{- if and (eq "http" $port_map.scheme) (not (hasKey $found "http")) -}}
+{{- $found := set $found "http" $port_map.name -}}
+{{- end -}}
+{{- if not (hasKey $found "any") -}}
+{{- $found := set $found "any" $port_map.name -}}
+{{- end -}}
+{{- end -}}
+{{- if hasKey $found "https-443" -}}
+{{- get $found "https-443" -}}
+{{- else if hasKey $found "http-80" -}}
+{{- get $found "http-80" -}}
+{{- else if hasKey $found "https" -}}
+{{- get $found "https" -}}
+{{- else if hasKey $found "http" -}}
+{{- get $found "http" -}}
+{{- else if hasKey $found "any" -}}
+{{- get $found "any" -}}
+{{- end -}}
+{{- end -}}

plugins/scheduler-k3s/templates/chart/certificate.yaml

@@ -30,7 +30,7 @@ spec:
       {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 6 }}
       {{ include "print.labels" (dict "config" $config "key" "secret") | indent 6 }}
   dnsNames:
-    {{- range $config.web.domains }}
-    - {{ . }}
+    {{- range $idx, $domain := $config.web.domains }}
+    - {{ $domain.name }}
     {{- end }}
 {{- end }}

plugins/scheduler-k3s/templates/chart/ingress-route.yaml

@@ -34,7 +34,7 @@ spec:
   routes:
     {{- range $ddx, $domain := $config.web.domains }}
     - kind: Rule
-      match: Host(`{{ $domain }}`)
+      match: Host(`{{ $domain.name }}`)
       {{- if $config.web.tls.enabled }}
       middlewares:
         - name: {{ $.Values.global.app_name}}-{{ $processName }}-redirect-to-https

plugins/scheduler-k3s/templates/chart/ingress.yaml

@@ -1,20 +1,14 @@
 {{- $processName := "PROCESS_NAME" }}
 {{- $config := index .Values.processes "PROCESS_NAME" }}
 {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }}
-{{- $mappings := dict }}
-{{- range $pdx, $port_map := $config.web.port_maps }}
-{{- $mappings := set $mappings $port_map.name "true" }}
-{{- end }}
-{{- range $pdx, $port_map := $config.web.port_maps }}
-{{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }}
-{{- continue }}
-{{- end }}
+{{- range $pdx, $domain := $config.web.domains }}
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
     dokku.com/managed: "true"
+    dokku.com/ingress-method: "domains"
     {{- if $config.web.tls.enabled }}
     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
     {{- end }}
@@ -26,30 +20,28 @@ metadata:
     app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
     {{ include "print.labels" (dict "config" $.Values.global "key" "ingress") | indent 4 }}
     {{ include "print.labels" (dict "config" $config "key" "ingress") | indent 4 }}
-  name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $port_map.name }}
+  name: {{ $.Values.global.app_name }}-{{ $processName }}-{{ $domain.slug }}
   namespace: {{ $.Values.global.namespace }}
 spec:
   ingressClassName: nginx
   {{- if $config.web.tls.enabled }}
   tls:
     - hosts:
-      {{- range $ddx, $domain := $config.web.domains }}
-      - {{ $domain | quote }}
-      {{- end }}
+      - {{ $domain.name | quote }}
       secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }}
   {{- end }}
   rules:
-    {{- range $ddx, $domain := $config.web.domains }}
-    - host: {{ $domain | quote }}
+    - host: {{ $domain.name | quote }}
       http:
         paths:
+          {{- if $config.web.port_maps }}
           - backend:
               service:
                 name: {{ $.Values.global.app_name }}-{{ $processName }}
                 port:
-                  name: {{ $port_map.name }}
+                  name: {{ include "primary.port" $config.web.port_maps }}
             pathType: ImplementationSpecific
             path: /
-    {{- end }}
+          {{- end }}
 {{- end }}
 {{- end }}

plugins/scheduler-k3s/triggers.go

@@ -23,6 +23,7 @@ import (
 	"github.com/dokku/dokku/plugins/config"
 	"github.com/dokku/dokku/plugins/cron"
 	"github.com/fatih/color"
+	"github.com/gosimple/slug"
 	"github.com/kballard/go-shellquote"
 	"github.com/ryanuber/columnize"
 	corev1 "k8s.io/api/core/v1"
@@ -36,7 +37,6 @@ func TriggerInstall() error {
 	if err := common.PropertySetup("scheduler-k3s"); err != nil {
 		return fmt.Errorf("Unable to install the scheduler-k3s plugin: %s", err.Error())
 	}
-
 	return nil
 }
 
@@ -385,8 +385,17 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		}
 
 		if processType == "web" {
+			sort.Strings(domains)
+			domainValues := []ProcessDomains{}
+			for _, domain := range domains {
+				domainValues = append(domainValues, ProcessDomains{
+					Name: domain,
+					Slug: slug.Make(domain),
+				})
+			}
+
 			processValues.Web = ProcessWeb{
-				Domains:  domains,
+				Domains:  domainValues,
 				PortMaps: []ProcessPortMap{},
 				TLS: ProcessTls{
 					Enabled:    tlsEnabled,
@@ -435,7 +444,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			}
 
 			sort.Sort(NameSorter(processValues.Web.PortMaps))
-			sort.Strings(processValues.Web.Domains)
 		}
 
 		values.Processes[processType] = processValues
@@ -587,7 +595,39 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
 	}
 
-	common.LogExclaim(fmt.Sprintf("Installing %s", appName))
+	ingresses, err := clientset.ListIngresses(ctx, ListIngressesInput{
+		Namespace:     namespace,
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/instance=%s-web", appName),
+	})
+	if err != nil {
+		return fmt.Errorf("Error listing ingresses: %w", err)
+	}
+
+	ingressesToDelete := []string{}
+
+	for _, ingress := range ingresses {
+		ingressIngressMethod := ingress.Annotations["dokku.com/ingress-method"]
+		if ingressIngressMethod != "domains" {
+			ingressesToDelete = append(ingressesToDelete, ingress.Name)
+		}
+	}
+
+	if len(ingressesToDelete) > 0 {
+		common.LogWarn("Manually removing non-matching ingress resources")
+	}
+	for _, ingressName := range ingressesToDelete {
+		common.LogVerboseQuiet(fmt.Sprintf("Removing non-matching ingress resource: %s", ingressName))
+		err := clientset.DeleteIngress(ctx, DeleteIngressInput{
+			Name:      ingressName,
+			Namespace: namespace,
+		})
+
+		if err != nil {
+			return fmt.Errorf("Error deleting ingress: %w", err)
+		}
+	}
+
+	common.LogInfo2(fmt.Sprintf("Installing %s", appName))
 	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
 		ChartPath:         chartPath,
 		Namespace:         namespace,

tests/unit/scheduler-k3s-1.bats

@@ -32,3 +32,33 @@ teardown_() {
 
   INGRESS_CLASS=nginx TAINT_SCHEDULING=true install_k3s
 }
+
+@test "(scheduler-k3s) deploy dockerfile exposed port" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:from-image $TEST_APP metabase/metabase:latest"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 30"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  assert_http_localhost_response_contains "http" "$TEST_APP.dokku.me" "80" "" "<title>Metabase</title>"
+}

tests/unit/test_helper.bash

@@ -238,6 +238,22 @@ assert_http_localhost_response() {
   fi
 }
 
+assert_http_localhost_response_contains() {
+  local scheme="$1" domain="$2" port="${3:-80}" path="${4:-}" content="${5:-}" status_code="${6:-200}"
+  run curl --connect-to "$domain:$port:localhost:$port" -kSso /dev/null -w "%{http_code}" "$scheme://$domain:$port$path"
+  echo "curl: curl --connect-to $domain:$port:localhost:$port -kSso /dev/null -w %{http_code} $scheme://$domain:$port$path"
+  echo "output: $output"
+  echo "status: $status"
+  assert_output "$status_code"
+
+  if [[ -n "$content" ]]; then
+    run curl --connect-to "$domain:$port:localhost:$port" -kSs "$scheme://$domain:$port$path"
+    echo "output: $output"
+    echo "status: $status"
+    assert_output_contains "$content"
+  fi
+}
+
 assert_ssl_domain() {
   local domain=$1
   assert_app_domain "${domain}"