diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index f66026081..372ce4d70 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -531,7 +531,7 @@ dokku scheduler-k3s:ensure-charts --force Alternatively, a comma separated list of chart names can be specified to only force install the specified charts: ```shell -dokku scheduler-k3s:ensure-charts --force --chart-names cert-manager +dokku scheduler-k3s:ensure-charts --charts cert-manager ``` ## Scheduler Interface @@ -575,7 +575,19 @@ The following Dokku functionality is not implemented at this time. ### Logging support -App logs for the `logs` command are fetched by Dokku from running containers via the `kubectl` cli. Persisting logs via Vector is not implemented at this time. Users may choose to configure the Vector Kubernetes integration directly by following [this guide](https://vector.dev/docs/setup/installation/platforms/kubernetes/). +App logs for the `logs` command are fetched by Dokku from running containers via the Kubernetes api. While the `k3s` scheduler does not integrate with the `logs:vector-*` subcommands, it does respect the global `vector-sink` logs property. When that property is set, the `scheduler-k3s:ensure-charts` command can be utilized to reconfigure vector to ship kubernetes logs to the provided sink. + +To setup log shipping, configure the global `vector-sink` property for the `logs` plugin.. Note that this can be run before _or_ after the `scheduler-k3s:ensure-charts` command - if run before, the `scheduler-k3s:ensure-charts` subcommand will pick up the value. + +```shell +dokku logs:set --global vector-sink "console://?encoding[codec]=json" +``` + +Next, run the `scheduler-k3s:ensure-charts` command with the `vector` chart to force the `k3s` scheduler to reconfigure vector with the specified sink: + +```shell +dokku scheduler-k3s:ensure-charts --charts vector +``` ### Supported Resource Management Properties diff --git a/plugins/logs/functions.go b/plugins/logs/functions.go index fa9fbf2c3..8908feecc 100644 --- a/plugins/logs/functions.go +++ b/plugins/logs/functions.go @@ -6,18 +6,16 @@ import ( "errors" "fmt" "html/template" - "net/url" "os" "path/filepath" "strings" "github.com/dokku/dokku/plugins/common" - "github.com/joncalhoun/qson" ) type vectorConfig struct { Sources map[string]vectorSource `json:"sources"` - Sinks map[string]vectorSink `json:"sinks"` + Sinks map[string]VectorSink `json:"sinks"` } type vectorSource struct { @@ -31,8 +29,6 @@ type vectorTemplateData struct { VectorImage string } -type vectorSink map[string]interface{} - const vectorContainerName = "vector-vector-1" const vectorOldContainerName = "vector" @@ -195,53 +191,11 @@ func stopVectorContainer() error { return nil } -func sinkValueToConfig(appName string, sinkValue string) (vectorSink, error) { - var data vectorSink - if strings.Contains(sinkValue, "://") { - parts := strings.SplitN(sinkValue, "://", 2) - parts[0] = strings.ReplaceAll(parts[0], "_", "-") - sinkValue = strings.Join(parts, "://") - } - u, err := url.Parse(sinkValue) - if err != nil { - return data, err - } - - if u.Query().Get("sinks") != "" { - return data, errors.New("Invalid option sinks") - } - - u.Scheme = strings.ReplaceAll(u.Scheme, "-", "_") - - query := u.RawQuery - query = strings.TrimPrefix(query, "&") - - b, err := qson.ToJSON(query) - if err != nil { - return data, err - } - - if err := json.Unmarshal(b, &data); err != nil { - return data, err - } - - data["type"] = u.Scheme - data["inputs"] = []string{"docker-source:" + appName} - if appName == "--global" { - data["inputs"] = []string{"docker-global-source"} - } - if appName == "--null" { - data["inputs"] = []string{"docker-null-source"} - } - - return data, nil -} - func writeVectorConfig() error { apps, _ := common.UnfilteredDokkuApps() data := vectorConfig{ Sources: map[string]vectorSource{}, - Sinks: map[string]vectorSink{}, + Sinks: map[string]VectorSink{}, } for _, appName := range apps { value := common.PropertyGet("logs", appName, "vector-sink") @@ -250,7 +204,7 @@ func writeVectorConfig() error { } inflectedAppName := strings.ReplaceAll(appName, ".", "-") - sink, err := sinkValueToConfig(inflectedAppName, value) + sink, err := SinkValueToConfig(inflectedAppName, value) if err != nil { return err } @@ -265,7 +219,7 @@ func writeVectorConfig() error { value := common.PropertyGet("logs", "--global", "vector-sink") if value != "" { - sink, err := sinkValueToConfig("--global", value) + sink, err := SinkValueToConfig("--global", value) if err != nil { return err } @@ -288,7 +242,7 @@ func writeVectorConfig() error { if len(data.Sinks) == 0 { // write logs to a blackhole - sink, err := sinkValueToConfig("--null", VectorDefaultSink) + sink, err := SinkValueToConfig("--null", VectorDefaultSink) if err != nil { return err } diff --git a/plugins/logs/logs.go b/plugins/logs/logs.go index 7e0f67fbb..15728b9a5 100644 --- a/plugins/logs/logs.go +++ b/plugins/logs/logs.go @@ -2,11 +2,19 @@ package logs import ( "embed" + "encoding/json" + "errors" "fmt" + "net/url" + "strings" "github.com/dokku/dokku/plugins/common" + "github.com/joncalhoun/qson" ) +// VectorSink is a map of vector sink properties +type VectorSink map[string]interface{} + // MaxSize is the default max retention size for docker logs const MaxSize = "10m" @@ -53,3 +61,46 @@ func GetFailedLogs(appName string) error { }) return err } + +// SinkValueToConfig converts a sink DSN value to a VectorSink +func SinkValueToConfig(appName string, sinkValue string) (VectorSink, error) { + var data VectorSink + if strings.Contains(sinkValue, "://") { + parts := strings.SplitN(sinkValue, "://", 2) + parts[0] = strings.ReplaceAll(parts[0], "_", "-") + sinkValue = strings.Join(parts, "://") + } + u, err := url.Parse(sinkValue) + if err != nil { + return data, err + } + + if u.Query().Get("sinks") != "" { + return data, errors.New("Invalid option sinks") + } + + u.Scheme = strings.ReplaceAll(u.Scheme, "-", "_") + + query := u.RawQuery + query = strings.TrimPrefix(query, "&") + + b, err := qson.ToJSON(query) + if err != nil { + return data, err + } + + if err := json.Unmarshal(b, &data); err != nil { + return data, err + } + + data["type"] = u.Scheme + data["inputs"] = []string{"docker-source:" + appName} + if appName == "--global" { + data["inputs"] = []string{"docker-global-source"} + } + if appName == "--null" { + data["inputs"] = []string{"docker-null-source"} + } + + return data, nil +} diff --git a/plugins/logs/set.go b/plugins/logs/set.go index bb87cf388..11e4945b8 100644 --- a/plugins/logs/set.go +++ b/plugins/logs/set.go @@ -49,7 +49,7 @@ func validateVectorSink(appName string, value string) error { return nil } - _, err := sinkValueToConfig(appName, value) + _, err := SinkValueToConfig(appName, value) if err != nil { return err } diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 50e5105e2..942d5c4f2 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -18,6 +18,7 @@ import ( appjson "github.com/dokku/dokku/plugins/app-json" "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/logs" nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts" resty "github.com/go-resty/resty/v2" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" @@ -1527,6 +1528,10 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn } } + if chart.ReleaseName == "vector" { + values = updateVectorValues(values) + } + chartProperties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "chart."+chart.ReleaseName+".") if err != nil { return fmt.Errorf("Error getting chart properties: %w", err) @@ -1566,6 +1571,28 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn return nil } +func updateVectorValues(values map[string]interface{}) map[string]interface{} { + value := common.PropertyGet("logs", "--global", "vector-sink") + if value == "" { + return values + } + + sink, err := logs.SinkValueToConfig("--global", value) + if err != nil { + return nil + } + + sink["inputs"] = []string{"kubernetes_container_logs"} + + sinkMap := map[string]interface{}{ + "kubernetes_global_sink": sink, + } + + values["customConfig"].(map[string]interface{})["sinks"] = sinkMap + + return values +} + func installHelperCommands(ctx context.Context) error { urls := map[string]string{ "kubectx": "https://github.com/ahmetb/kubectx/releases/latest/download/kubectx", diff --git a/plugins/scheduler-k3s/go.mod b/plugins/scheduler-k3s/go.mod index de4c0aecf..0c5044a31 100644 --- a/plugins/scheduler-k3s/go.mod +++ b/plugins/scheduler-k3s/go.mod @@ -8,6 +8,7 @@ require ( github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/config v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000 + github.com/dokku/dokku/plugins/logs v0.0.0-00010101000000-000000000000 github.com/dokku/dokku/plugins/nginx-vhosts v0.0.0-00010101000000-000000000000 github.com/fatih/color v1.18.0 github.com/go-openapi/jsonpointer v0.21.0 @@ -59,6 +60,7 @@ require ( github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect + github.com/dokku/dokku/plugins/docker-options v0.0.0-00010101000000-000000000000 // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -96,6 +98,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect github.com/joho/godotenv v1.5.1 // indirect + github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11-0.20241004063537-dbd6c381492a // indirect @@ -194,6 +197,10 @@ replace github.com/dokku/dokku/plugins/config => ../config replace github.com/dokku/dokku/plugins/cron => ../cron +replace github.com/dokku/dokku/plugins/docker-options => ../docker-options + +replace github.com/dokku/dokku/plugins/logs => ../logs + replace github.com/dokku/dokku/plugins/nginx-vhosts => ../nginx-vhosts replace github.com/joho/godotenv => github.com/joho/godotenv v1.2.0 diff --git a/plugins/scheduler-k3s/go.sum b/plugins/scheduler-k3s/go.sum index 82dcffa48..02097d141 100644 --- a/plugins/scheduler-k3s/go.sum +++ b/plugins/scheduler-k3s/go.sum @@ -221,6 +221,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.2.0 h1:vGTvz69FzUFp+X4/bAkb0j5BoLC+9bpqTWY8mjhA9pc= github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0 h1:ct2XA1aDw8A07Dr8gtrrZgIgLKcZNAl2o9nn0WRMK4Y= +github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0/go.mod h1:DFXrEwSRX0p/aSvxE21319menCBFeQO0jXpRj7LEZUA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 811618cbd..9a37a78bd 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -132,6 +132,14 @@ var HelmCharts = []HelmChart{ RepoURL: "https://kedacore.github.io/charts", Version: "0.8.0", }, + { + ChartPath: "vector", + CreateNamespace: true, + Namespace: "vector", + ReleaseName: "vector", + RepoURL: "https://helm.vector.dev", + Version: "0.42.0", + }, } type HelmRepository struct { diff --git a/plugins/scheduler-k3s/templates/helm-config/vector.yaml b/plugins/scheduler-k3s/templates/helm-config/vector.yaml new file mode 100644 index 000000000..e20ccf829 --- /dev/null +++ b/plugins/scheduler-k3s/templates/helm-config/vector.yaml @@ -0,0 +1,42 @@ +--- +role: Agent + +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/infra + operator: Exists + +service: + ports: + - name: prom-exporter + port: 9090 + protocol: TCP + +customConfig: + data_dir: /vector-data-dir + + sources: + kubernetes_logs: + type: kubernetes_logs + + transforms: + kubernetes_container_logs: + type: remap + inputs: + - kubernetes_logs + source: | + .message = .message + .container = .kubernetes.container_name + .pod = .kubernetes.pod_name + .namespace = .kubernetes.pod_namespace + + sinks: + default_global_sink: + type: console + inputs: + - kubernetes_container_logs + encoding: + codec: json