From bfccf60a37314c20ed654200d7bbc559c97a56d1 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 11 Feb 2024 07:37:47 -0500 Subject: [PATCH 1/3] refactor: simplify tls handling by always redirecting --- plugins/scheduler-k3s/template.go | 11 +++++------ .../templates/chart/ingress-route.yaml | 9 ++++----- .../scheduler-k3s/templates/chart/ingress.yaml | 5 ++++- plugins/scheduler-k3s/triggers.go | 15 +++++---------- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index c371daae8..3addd48d3 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -161,12 +161,11 @@ type ProcessCron struct { } type ProcessPortMap struct { - ContainerPort int32 `yaml:"container_port"` - HostPort int32 `yaml:"host_port"` - Scheme string `yaml:"scheme"` - Protocol PortmapProtocol `yaml:"protocol"` - Name string `yaml:"name"` - RedirectToHttps bool `yaml:"redirect_to_https"` + ContainerPort int32 `yaml:"container_port"` + HostPort int32 `yaml:"host_port"` + Scheme string `yaml:"scheme"` + Protocol PortmapProtocol `yaml:"protocol"` + Name string `yaml:"name"` } type PortmapProtocol string diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index ca9e3f87e..f5fcbb96c 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -26,16 +26,15 @@ metadata: namespace: {{ $.Values.global.namespace }} spec: entryPoints: - {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }} + {{- if $config.web.tls.enabled }} - websecure - {{- else }} - - web {{- end }} + - web routes: {{- range $ddx, $domain := $config.web.domains }} - kind: Rule match: Host(`{{ $domain }}`) - {{- if and $config.web.tls.enabled (and (eq $port_map.scheme "http") $port_map.redirect_to_https) }} + {{- if $config.web.tls.enabled }} middlewares: - name: {{ $.Values.global.app_name}}-{{ $processName }}-redirect-to-https namespace: {{ $.Values.global.namespace }} @@ -47,7 +46,7 @@ spec: port: {{ $port_map.name }} scheme: http {{- end }} - {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }} + {{- if $config.web.tls.enabled }} tls: secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index 36a622a8b..414022b07 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -18,6 +18,9 @@ metadata: {{ $k }}: {{ $v | quote }} {{- end }} {{- end }} + {{- if $config.web.tls.enabled }} + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + {{- end }} labels: app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $processName }} app.kubernetes.io/name: {{ $processName }} @@ -26,7 +29,7 @@ metadata: namespace: {{ $.Values.global.namespace }} spec: ingressClassName: nginx - {{- if and $config.web.tls.enabled (eq $port_map.scheme "https") }} + {{- if $config.web.tls.enabled }} tls: - hosts: {{- range $ddx, $domain := $config.web.domains }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 16f824271..51caafee4 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -417,17 +417,12 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e protocol = PortmapProtocol_UDP } - redirecToHttps := false - if _, ok := portMaps[fmt.Sprintf("https-443-%d", portMap.ContainerPort)]; ok && portMap.Scheme == "http" { - redirecToHttps = true - } processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{ - ContainerPort: portMap.ContainerPort, - HostPort: portMap.HostPort, - Name: portMap.String(), - Protocol: protocol, - RedirectToHttps: redirecToHttps, - Scheme: portMap.Scheme, + ContainerPort: portMap.ContainerPort, + HostPort: portMap.HostPort, + Name: portMap.String(), + Protocol: protocol, + Scheme: portMap.Scheme, }) } From 433eb26f3ccbf35542e8dce5a1b5762623d3b81f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 11 Feb 2024 07:38:42 -0500 Subject: [PATCH 2/3] feat: always skip the https ingress/ingress-route/deployment port entry if there is a corresponding http entry Also inject the corresponding entries as necessary --- docs/deployment/schedulers/k3s.md | 2 +- .../templates/chart/deployment.yaml | 7 ++++++ .../templates/chart/ingress-route.yaml | 7 ++++++ .../templates/chart/ingress.yaml | 7 ++++++ plugins/scheduler-k3s/triggers.go | 24 +++++++++++++++++++ 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 540bd157d..e17a25fc7 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -243,7 +243,7 @@ dokku scheduler-k3s:set --global letsencrypt-email-prod automated@dokku.sh dokku scheduler-k3s:set --global letsencrypt-email-stag automated@dokku.sh ``` -After enabling and rebuilding, all apps with an `https` port mapping that utilize the related letsencrypt server will be automatically updated to enable ssl. All http requests will then be redirected to https. +After enabling and rebuilding, all apps with an `http:80` port mapping will have a corresponding `https:443` added and ssl will be automatically enabled. All http requests will then be redirected to https. ### Customizing the letsencrypt server diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index d064996fd..db957375e 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -1,5 +1,9 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} +{{- $mappings := dict }} +{{- range $pdx, $port_map := $config.web.port_maps }} +{{- $mappings := set $mappings $port_map.name "true" }} +{{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -76,6 +80,9 @@ spec: {{- if eq $processName "web" }} ports: {{- range $pdx, $port_map := $config.web.port_maps }} + {{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }} + {{- continue }} + {{- end }} - containerPort: {{ $port_map.container_port }} name: {{ $port_map.name }} protocol: {{ $port_map.protocol }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index f5fcbb96c..578fa7b3c 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -1,7 +1,14 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "traefik") }} +{{- $mappings := dict }} {{- range $pdx, $port_map := $config.web.port_maps }} +{{- $mappings := set $mappings $port_map.name "true" }} +{{- end }} +{{- range $pdx, $port_map := $config.web.port_maps }} +{{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }} +{{- continue }} +{{- end }} --- apiVersion: traefik.io/v1alpha1 kind: IngressRoute diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index 414022b07..4509cc5e2 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -1,7 +1,14 @@ {{- $processName := "PROCESS_NAME" }} {{- $config := index .Values.processes "PROCESS_NAME" }} {{- if and $config.web.domains (eq $.Values.global.network.ingress_class "nginx") }} +{{- $mappings := dict }} {{- range $pdx, $port_map := $config.web.port_maps }} +{{- $mappings := set $mappings $port_map.name "true" }} +{{- end }} +{{- range $pdx, $port_map := $config.web.port_maps }} +{{- if and (eq $port_map.scheme "https") (hasKey $mappings (printf "http-80-%.0f" $port_map.container_port)) }} +{{- continue }} +{{- end }} --- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 51caafee4..b89be6147 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -426,6 +426,30 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e }) } + for _, portMap := range processValues.Web.PortMaps { + _, httpOk := portMaps[fmt.Sprintf("http-80-%d", portMap.ContainerPort)] + _, httpsOk := portMaps[fmt.Sprintf("https-443-%d", portMap.ContainerPort)] + if portMap.Scheme == "http" && !httpsOk && tlsEnabled { + processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{ + ContainerPort: portMap.ContainerPort, + HostPort: 443, + Name: fmt.Sprintf("https-443-%d", portMap.ContainerPort), + Protocol: PortmapProtocol_TCP, + Scheme: "https", + }) + } + + if portMap.Scheme == "https" && !httpOk { + processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{ + ContainerPort: portMap.ContainerPort, + HostPort: 80, + Name: fmt.Sprintf("http-80-%d", portMap.ContainerPort), + Protocol: PortmapProtocol_TCP, + Scheme: "http", + }) + } + } + sort.Sort(NameSorter(processValues.Web.PortMaps)) sort.Strings(processValues.Web.Domains) } From abbc13cb857eacdf0cfe30bed152de3ed0108e85 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 12 Feb 2024 01:30:17 -0500 Subject: [PATCH 3/3] fix: ensure the ingress class is properly set for certificate usage Without this, switching to nginx ingress fails as the cluster issuer isn't properly configured --- plugins/scheduler-k3s/functions.go | 20 +++++++++++--------- plugins/scheduler-k3s/triggers.go | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 5571cb19e..540d922a3 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -149,16 +149,18 @@ func applyClusterIssuers(ctx context.Context) error { clusterIssuerValues := ClusterIssuerValues{ ClusterIssuers: map[string]ClusterIssuer{ "letsencrypt-stag": { - Email: letsencryptEmailStag, - Enabled: letsencryptEmailStag != "", - Name: "letsencrypt-stag", - Server: "https://acme-staging-v02.api.letsencrypt.org/directory", + Email: letsencryptEmailStag, + Enabled: letsencryptEmailStag != "", + IngressClass: getGlobalIngressClass(), + Name: "letsencrypt-stag", + Server: "https://acme-staging-v02.api.letsencrypt.org/directory", }, "letsencrypt-prod": { - Email: letsencryptEmailProd, - Enabled: letsencryptEmailProd != "", - Name: "letsencrypt-prod", - Server: "https://acme-v02.api.letsencrypt.org/directory", + Email: letsencryptEmailProd, + Enabled: letsencryptEmailProd != "", + IngressClass: getGlobalIngressClass(), + Name: "letsencrypt-prod", + Server: "https://acme-v02.api.letsencrypt.org/directory", }, }, } @@ -496,7 +498,7 @@ func getComputedImagePullSecrets(appName string) string { } func getGlobalIngressClass() string { - return common.PropertyGetDefault("scheduler-k3s", "global", "ingress-class", DefaultIngressClass) + return common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass) } func getIngressAnnotations(appName string, processType string) (map[string]string, error) { diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index b89be6147..f566e32c8 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -351,7 +351,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e }, Namespace: namespace, Network: GlobalNetwork{ - IngressClass: common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass), + IngressClass: getGlobalIngressClass(), PrimaryPort: primaryPort, }, Secrets: map[string]string{},