From 9c41bed78b6ffe1e2db263d335e604a55886a566 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 23 Aug 2025 15:46:57 -0400 Subject: [PATCH 1/3] feat: allow specifying base64-encoded values in vector-sink DSN urls This will allow Vector templating to work when using Kubernetes as the scheduler. --- docs/deployment/logs.md | 13 +++++++++++++ docs/deployment/schedulers/k3s.md | 1 + plugins/logs/logs.go | 17 +++++++++++++++++ tests/unit/scheduler-k3s-3.bats | 31 +++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/docs/deployment/logs.md b/docs/deployment/logs.md index 33ff6953e..ee58053f9 100644 --- a/docs/deployment/logs.md +++ b/docs/deployment/logs.md @@ -234,6 +234,19 @@ For some sinks - such as the `http` sink - it may be useful to use special chara dokku logs:set test vector-sink "http://?uri=https%3A//loggerservice.com%3A1234/%3Ftoken%3Dabc1234%26type%3Dvector" ``` +For kubernetes, it may be necessary to use template syntax - `{{ .parent.child }}` - in the sink configuration. Naively using brackets will fail due to the Helm chart install process assuming that the template should be interpreted at Helm install time vs by Vector itself. To avoid this, use `base64enc:` values (available only for top-level properties at this time). The following example shows how to use `{{ pod }}` as a value. + +```shell +# encode the value with a Helm `print` statement wrapper +encoded="$(echo '{{ print "{{ pod }}" }}' | base64)" +# the value of encoded should be: e3sgcHJpbnQgInt7IHBvZCB9fSIgfX0K + +# set the value +dokku logs:set test vector-sink "http://?process=base64enc%3A${encoded}" +``` + +This will transform the value to it's encoded form when configuring Vector sinks for Kubernetes. + Please read the [sink documentation](https://vector.dev/docs/reference/configuration/sinks/) for your sink of choice to configure the sink as desired. ##### Configuring the app label diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 5056deb2e..1f874c8c1 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -611,6 +611,7 @@ Next, run the `scheduler-k3s:ensure-charts` command with the `vector` chart to f ```shell dokku scheduler-k3s:ensure-charts --charts vector ``` +Please see the [vector logs documentation](/docs/deployment/logs.md#configuring-a-log-sink) for more information on specifying vector sinks. ### Supported Resource Management Properties diff --git a/plugins/logs/logs.go b/plugins/logs/logs.go index f68508c57..8a21bbc84 100644 --- a/plugins/logs/logs.go +++ b/plugins/logs/logs.go @@ -2,6 +2,7 @@ package logs import ( "embed" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -102,5 +103,21 @@ func SinkValueToConfig(appName string, sinkValue string) (VectorSink, error) { data["inputs"] = []string{"docker-null-source"} } + // add special support for `base64enc:VAL` fields + for key, value := range data { + valueString, ok := value.(string) + if !ok { + continue + } + + if encodedValue, found := strings.CutPrefix(valueString, "base64enc:"); found { + decodedValue, err := base64.StdEncoding.DecodeString(encodedValue) + if err != nil { + return data, fmt.Errorf("Error decoding base64: %w", err) + } + data[key] = strings.TrimSpace(string(decodedValue)) + } + } + return data, nil } diff --git a/tests/unit/scheduler-k3s-3.bats b/tests/unit/scheduler-k3s-3.bats index 29d0f2f27..7a0d555c0 100644 --- a/tests/unit/scheduler-k3s-3.bats +++ b/tests/unit/scheduler-k3s-3.bats @@ -52,6 +52,37 @@ teardown_() { assert_output "2" } +@test "(scheduler-k3s) deploy kustomize with vector sink" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + encoded="$(echo '{{ print "{{ pod }}" }}' | base64)" + run /bin/bash -c "echo $encoded" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "e3sgcHJpbnQgInt7IHBvZCB9fSIgfX0K" + + run /bin/bash -c "dokku logs:set --global vector-sink 'http://?process=base64enc%3A${encoded}'" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:ensure-charts --charts vector" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get cm -n vector vector -o yaml" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "process: '{{ pod }}'" +} + inject_kustomization_yaml() { local APP="$1" local APP_REPO_DIR="$2" From fd61fdd2ab905d3fa4c3b2dde2f0aa91b80ce79c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 23 Aug 2025 16:14:51 -0400 Subject: [PATCH 2/3] fix: ensure the vector values are only set for the vector chart --- plugins/scheduler-k3s/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index a2084c305..afcc79117 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1577,7 +1577,7 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn } } - if chart.ReleaseName == "vector" { + if chart.ReleaseName == "vector" && chart.Namespace == "vector" { values = updateVectorValues(values) } From 3f12ce2e8d7780895123909098a43d9e9cedc0d7 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 23 Aug 2025 16:15:00 -0400 Subject: [PATCH 3/3] refactor: simplify test --- tests/unit/scheduler-k3s-3.bats | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/unit/scheduler-k3s-3.bats b/tests/unit/scheduler-k3s-3.bats index 7a0d555c0..4a6a3fb07 100644 --- a/tests/unit/scheduler-k3s-3.bats +++ b/tests/unit/scheduler-k3s-3.bats @@ -57,8 +57,6 @@ teardown_() { skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" fi - INGRESS_CLASS=nginx install_k3s - encoded="$(echo '{{ print "{{ pod }}" }}' | base64)" run /bin/bash -c "echo $encoded" echo "output: $output" @@ -71,10 +69,7 @@ teardown_() { echo "status: $status" assert_success - run /bin/bash -c "dokku scheduler-k3s:ensure-charts --charts vector" - echo "output: $output" - echo "status: $status" - assert_success + INGRESS_CLASS=nginx install_k3s run /bin/bash -c "kubectl get cm -n vector vector -o yaml" echo "output: $output"