Merge pull request #7648 from dokku/k3s-vector-integration

feat: add ability to ship k3s container logs via vector

docs/deployment/schedulers/k3s.md

@@ -531,7 +531,7 @@ dokku scheduler-k3s:ensure-charts --force
 Alternatively, a comma separated list of chart names can be specified to only force install the specified charts:
 
 ```shell
-dokku scheduler-k3s:ensure-charts --force --chart-names cert-manager
+dokku scheduler-k3s:ensure-charts --charts cert-manager
 ```
 
 ## Scheduler Interface
@@ -575,7 +575,19 @@ The following Dokku functionality is not implemented at this time.
 
 ### Logging support
 
-App logs for the `logs` command are fetched by Dokku from running containers via the `kubectl` cli. Persisting logs via Vector is not implemented at this time. Users may choose to configure the Vector Kubernetes integration directly by following [this guide](https://vector.dev/docs/setup/installation/platforms/kubernetes/).
+App logs for the `logs` command are fetched by Dokku from running containers via the Kubernetes api. While the `k3s` scheduler does not integrate with the `logs:vector-*` subcommands, it does respect the global `vector-sink` logs property. When that property is set, the `scheduler-k3s:ensure-charts` command can be utilized to reconfigure vector to ship kubernetes logs to the provided sink.
+
+To setup log shipping, configure the global `vector-sink` property for the `logs` plugin.. Note that this can be run before _or_ after the `scheduler-k3s:ensure-charts` command - if run before, the `scheduler-k3s:ensure-charts` subcommand will pick up the value.
+
+```shell
+dokku logs:set --global vector-sink "console://?encoding[codec]=json"
+```
+
+Next, run the `scheduler-k3s:ensure-charts` command with the `vector` chart to force the `k3s` scheduler to reconfigure vector with the specified sink:
+
+```shell
+dokku scheduler-k3s:ensure-charts --charts vector
+```
 
 ### Supported Resource Management Properties
 

plugins/logs/functions.go

@@ -6,18 +6,16 @@ import (
 	"errors"
 	"fmt"
 	"html/template"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/dokku/dokku/plugins/common"
-	"github.com/joncalhoun/qson"
 )
 
 type vectorConfig struct {
 	Sources map[string]vectorSource `json:"sources"`
-	Sinks   map[string]vectorSink   `json:"sinks"`
+	Sinks   map[string]VectorSink   `json:"sinks"`
 }
 
 type vectorSource struct {
@@ -31,8 +29,6 @@ type vectorTemplateData struct {
 	VectorImage  string
 }
 
-type vectorSink map[string]interface{}
-
 const vectorContainerName = "vector-vector-1"
 const vectorOldContainerName = "vector"
 
@@ -195,53 +191,11 @@ func stopVectorContainer() error {
 	return nil
 }
 
-func sinkValueToConfig(appName string, sinkValue string) (vectorSink, error) {
-	var data vectorSink
-	if strings.Contains(sinkValue, "://") {
-		parts := strings.SplitN(sinkValue, "://", 2)
-		parts[0] = strings.ReplaceAll(parts[0], "_", "-")
-		sinkValue = strings.Join(parts, "://")
-	}
-	u, err := url.Parse(sinkValue)
-	if err != nil {
-		return data, err
-	}
-
-	if u.Query().Get("sinks") != "" {
-		return data, errors.New("Invalid option sinks")
-	}
-
-	u.Scheme = strings.ReplaceAll(u.Scheme, "-", "_")
-
-	query := u.RawQuery
-	query = strings.TrimPrefix(query, "&")
-
-	b, err := qson.ToJSON(query)
-	if err != nil {
-		return data, err
-	}
-
-	if err := json.Unmarshal(b, &data); err != nil {
-		return data, err
-	}
-
-	data["type"] = u.Scheme
-	data["inputs"] = []string{"docker-source:" + appName}
-	if appName == "--global" {
-		data["inputs"] = []string{"docker-global-source"}
-	}
-	if appName == "--null" {
-		data["inputs"] = []string{"docker-null-source"}
-	}
-
-	return data, nil
-}
-
 func writeVectorConfig() error {
 	apps, _ := common.UnfilteredDokkuApps()
 	data := vectorConfig{
 		Sources: map[string]vectorSource{},
-		Sinks:   map[string]vectorSink{},
+		Sinks:   map[string]VectorSink{},
 	}
 	for _, appName := range apps {
 		value := common.PropertyGet("logs", appName, "vector-sink")
@@ -250,7 +204,7 @@ func writeVectorConfig() error {
 		}
 
 		inflectedAppName := strings.ReplaceAll(appName, ".", "-")
-		sink, err := sinkValueToConfig(inflectedAppName, value)
+		sink, err := SinkValueToConfig(inflectedAppName, value)
 		if err != nil {
 			return err
 		}
@@ -265,7 +219,7 @@ func writeVectorConfig() error {
 
 	value := common.PropertyGet("logs", "--global", "vector-sink")
 	if value != "" {
-		sink, err := sinkValueToConfig("--global", value)
+		sink, err := SinkValueToConfig("--global", value)
 		if err != nil {
 			return err
 		}
@@ -288,7 +242,7 @@ func writeVectorConfig() error {
 
 	if len(data.Sinks) == 0 {
 		// write logs to a blackhole
-		sink, err := sinkValueToConfig("--null", VectorDefaultSink)
+		sink, err := SinkValueToConfig("--null", VectorDefaultSink)
 		if err != nil {
 			return err
 		}

plugins/logs/logs.go

@@ -2,11 +2,19 @@ package logs
 
 import (
 	"embed"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"net/url"
+	"strings"
 
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/joncalhoun/qson"
 )
 
+// VectorSink is a map of vector sink properties
+type VectorSink map[string]interface{}
+
 // MaxSize is the default max retention size for docker logs
 const MaxSize = "10m"
 
@@ -53,3 +61,46 @@ func GetFailedLogs(appName string) error {
 	})
 	return err
 }
+
+// SinkValueToConfig converts a sink DSN value to a VectorSink
+func SinkValueToConfig(appName string, sinkValue string) (VectorSink, error) {
+	var data VectorSink
+	if strings.Contains(sinkValue, "://") {
+		parts := strings.SplitN(sinkValue, "://", 2)
+		parts[0] = strings.ReplaceAll(parts[0], "_", "-")
+		sinkValue = strings.Join(parts, "://")
+	}
+	u, err := url.Parse(sinkValue)
+	if err != nil {
+		return data, err
+	}
+
+	if u.Query().Get("sinks") != "" {
+		return data, errors.New("Invalid option sinks")
+	}
+
+	u.Scheme = strings.ReplaceAll(u.Scheme, "-", "_")
+
+	query := u.RawQuery
+	query = strings.TrimPrefix(query, "&")
+
+	b, err := qson.ToJSON(query)
+	if err != nil {
+		return data, err
+	}
+
+	if err := json.Unmarshal(b, &data); err != nil {
+		return data, err
+	}
+
+	data["type"] = u.Scheme
+	data["inputs"] = []string{"docker-source:" + appName}
+	if appName == "--global" {
+		data["inputs"] = []string{"docker-global-source"}
+	}
+	if appName == "--null" {
+		data["inputs"] = []string{"docker-null-source"}
+	}
+
+	return data, nil
+}

plugins/logs/set.go

@@ -49,7 +49,7 @@ func validateVectorSink(appName string, value string) error {
 		return nil
 	}
 
-	_, err := sinkValueToConfig(appName, value)
+	_, err := SinkValueToConfig(appName, value)
 	if err != nil {
 		return err
 	}

plugins/scheduler-k3s/functions.go

@@ -18,6 +18,7 @@ import (
 
 	appjson "github.com/dokku/dokku/plugins/app-json"
 	"github.com/dokku/dokku/plugins/common"
+	"github.com/dokku/dokku/plugins/logs"
 	nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts"
 	resty "github.com/go-resty/resty/v2"
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
@@ -1527,6 +1528,10 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 			}
 		}
 
+		if chart.ReleaseName == "vector" {
+			values = updateVectorValues(values)
+		}
+
 		chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart."+chart.ReleaseName+".")
 		if err != nil {
 			return fmt.Errorf("Error getting chart properties: %w", err)
@@ -1566,6 +1571,28 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
 	return nil
 }
 
+func updateVectorValues(values map[string]interface{}) map[string]interface{} {
+	value := common.PropertyGet("logs", "--global", "vector-sink")
+	if value == "" {
+		return values
+	}
+
+	sink, err := logs.SinkValueToConfig("--global", value)
+	if err != nil {
+		return nil
+	}
+
+	sink["inputs"] = []string{"kubernetes_container_logs"}
+
+	sinkMap := map[string]interface{}{
+		"kubernetes_global_sink": sink,
+	}
+
+	values["customConfig"].(map[string]interface{})["sinks"] = sinkMap
+
+	return values
+}
+
 func installHelperCommands(ctx context.Context) error {
 	urls := map[string]string{
 		"kubectx": "https://github.com/ahmetb/kubectx/releases/latest/download/kubectx",

plugins/scheduler-k3s/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
+	github.com/dokku/dokku/plugins/logs v0.0.0-00010101000000-000000000000
 	github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-00010101000000-000000000000
 	github.com/fatih/color v1.18.0
 	github.com/go-openapi/jsonpointer v0.21.0
@@ -59,6 +60,7 @@ require (
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/dokku/dokku/plugins/docker-options v0.0.0-00010101000000-000000000000 // indirect
 	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
 	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
@@ -96,6 +98,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11-0.20241004063537-dbd6c381492a // indirect
@@ -194,6 +197,10 @@ replace github.com/dokku/dokku/plugins/config => ../config
 
 replace github.com/dokku/dokku/plugins/cron => ../cron
 
+replace github.com/dokku/dokku/plugins/docker-options => ../docker-options
+
+replace github.com/dokku/dokku/plugins/logs => ../logs
+
 replace github.com/dokku/dokku/plugins/nginx-vhosts => ../nginx-vhosts
 
 replace github.com/joho/godotenv => github.com/joho/godotenv v1.2.0

plugins/scheduler-k3s/go.sum

@@ -221,6 +221,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/joho/godotenv v1.2.0 h1:vGTvz69FzUFp+X4/bAkb0j5BoLC+9bpqTWY8mjhA9pc=
 github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0 h1:ct2XA1aDw8A07Dr8gtrrZgIgLKcZNAl2o9nn0WRMK4Y=
+github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0/go.mod h1:DFXrEwSRX0p/aSvxE21319menCBFeQO0jXpRj7LEZUA=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=

plugins/scheduler-k3s/scheduler_k3s.go

@@ -132,6 +132,14 @@ var HelmCharts = []HelmChart{
 		RepoURL:         "https://kedacore.github.io/charts",
 		Version:         "0.8.0",
 	},
+	{
+		ChartPath:       "vector",
+		CreateNamespace: true,
+		Namespace:       "vector",
+		ReleaseName:     "vector",
+		RepoURL:         "https://helm.vector.dev",
+		Version:         "0.42.0",
+	},
 }
 
 type HelmRepository struct {

plugins/scheduler-k3s/templates/helm-config/vector.yaml

@@ -0,0 +1,42 @@
+---
+role: Agent
+
+tolerations:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
+    operator: Exists
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/infra
+    operator: Exists
+
+service:
+  ports:
+  - name: prom-exporter
+    port: 9090
+    protocol: TCP
+
+customConfig:
+  data_dir: /vector-data-dir
+
+  sources:
+    kubernetes_logs:
+      type: kubernetes_logs
+
+  transforms:
+    kubernetes_container_logs:
+      type: remap
+      inputs:
+        - kubernetes_logs
+      source: |
+        .message = .message
+        .container = .kubernetes.container_name
+        .pod = .kubernetes.pod_name
+        .namespace = .kubernetes.pod_namespace
+
+  sinks:
+    default_global_sink:
+      type: console
+      inputs:
+        - kubernetes_container_logs
+      encoding:
+        codec: json