From c74c542b1fd0664ee01409eab3aa7e186ff1e022 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 20 Oct 2021 14:25:31 -0400 Subject: [PATCH 1/4] feat: upgrade vector and set default sink variable --- plugins/logs/logs.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/logs/logs.go b/plugins/logs/logs.go index 2b5e5f01c..7e2df356f 100644 --- a/plugins/logs/logs.go +++ b/plugins/logs/logs.go @@ -24,7 +24,10 @@ var ( ) // VectorImage contains the default vector image to run -const VectorImage = "timberio/vector:0.16.X-debian" +const VectorImage = "timberio/vector:0.17.X-debian" + +// VectorDefaultSink contains the default sink in use for vector log shipping +const VectorDefaultSink = "blackhole://?print_interval_secs=1" // GetFailedLogs outputs failed deploy logs for a given app func GetFailedLogs(appName string) error { From 01e317259bb737828d060ce63ff4a9c761f6a589 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Wed, 20 Oct 2021 14:26:30 -0400 Subject: [PATCH 2/4] refactor: use default sink variable This allows us to update the sink at the same time as updating the vector integration. --- plugins/logs/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/logs/functions.go b/plugins/logs/functions.go index 74f2b5146..50aa6a2ca 100644 --- a/plugins/logs/functions.go +++ b/plugins/logs/functions.go @@ -205,7 +205,7 @@ func writeVectorConfig() error { if len(data.Sinks) == 0 { // write logs to a blackhole - sink, err := sinkValueToConfig("--null", "blackhole://?print_amount=1") + sink, err := sinkValueToConfig("--null", VectorDefaultSink) if err != nil { return err } From 760be655e914898a164736cca8a82861bc79fe4c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 24 Oct 2021 20:20:35 -0400 Subject: [PATCH 3/4] fix: handle both stderr and stdout when tailing container logs Previously, stderr was ignored when tailing container log output in golang. This change ensures we capture it and also use the correct helper for that output method. --- plugins/common/log.go | 91 ++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/plugins/common/log.go b/plugins/common/log.go index 7cfc87b62..d45f26e48 100644 --- a/plugins/common/log.go +++ b/plugins/common/log.go @@ -1,11 +1,11 @@ package common import ( - "bufio" "fmt" "os" "strconv" "strings" + "sync" ) // ErrWithExitCode wraps error and exposes an ExitCode method @@ -13,6 +13,37 @@ type ErrWithExitCode interface { ExitCode() int } +type writer struct { + mu *sync.Mutex + source string +} + +// Write prints the data to either stdout or stderr using the log helper functions +func (w *writer) Write(bytes []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + if w.source == "stdout" { + for _, line := range strings.Split(string(bytes), "\n") { + if line == "" { + continue + } + + LogVerboseQuiet(line) + } + } else { + for _, line := range strings.Split(string(bytes), "\n") { + if line == "" { + continue + } + + LogVerboseStderrQuiet(line) + } + } + + return len(bytes), nil +} + // LogFail is the failure log formatter // prints text to stderr and exits with status 1 func LogFail(text string) { @@ -81,6 +112,12 @@ func LogVerbose(text string) { fmt.Println(fmt.Sprintf(" %s", text)) } +// LogVerboseStderr is the verbose log formatter +// prints indented text to stderr +func LogVerboseStderr(text string) { + fmt.Fprintln(os.Stderr, fmt.Sprintf(" ! %s", text)) +} + // LogVerboseQuiet is the verbose log formatter // prints indented text to stdout (with quiet option) func LogVerboseQuiet(text string) { @@ -89,25 +126,17 @@ func LogVerboseQuiet(text string) { } } +// LogVerboseStderrQuiet is the verbose log formatter +// prints indented text to stderr (with quiet option) +func LogVerboseStderrQuiet(text string) { + if os.Getenv("DOKKU_QUIET_OUTPUT") == "" { + LogVerboseStderr(text) + } +} + // LogVerboseQuietContainerLogs is the verbose log formatter for container logs func LogVerboseQuietContainerLogs(containerID string) { - sc := NewShellCmdWithArgs(DockerBin(), "container", "logs", containerID) - sc.ShowOutput = false - b, err := sc.CombinedOutput() - if err != nil { - LogExclaim(fmt.Sprintf("Failed to fetch container logs: %s", containerID)) - } - - output := strings.TrimSpace(string(b)) - if len(output) == 0 { - return - } - - for _, line := range strings.Split(output, "\n") { - if line != "" { - LogVerboseQuiet(line) - } - } + LogVerboseQuietContainerLogsTail(containerID, 0, false) } // LogVerboseQuietContainerLogsTail is the verbose log formatter for container logs with tail mode enabled @@ -119,17 +148,25 @@ func LogVerboseQuietContainerLogsTail(containerID string, lines int, tail bool) if tail { args = append(args, "--follow") } - sc := NewShellCmdWithArgs(DockerBin(), args...) - stdout, _ := sc.Command.StdoutPipe() - sc.Command.Start() - scanner := bufio.NewScanner(stdout) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - m := scanner.Text() - LogVerboseQuiet(m) + sc := NewShellCmdWithArgs(DockerBin(), args...) + var mu sync.Mutex + sc.Command.Stdout = &writer{ + mu: &mu, + source: "stdout", + } + sc.Command.Stderr = &writer{ + mu: &mu, + source: "stderr", + } + + if err := sc.Command.Start(); err != nil { + LogExclaim(fmt.Sprintf("Failed to fetch container logs: %s", containerID)) + } + + if err := sc.Command.Wait(); err != nil { + LogExclaim(fmt.Sprintf("Failed to fetch container logs: %s", containerID)) } - sc.Command.Wait() } // LogWarn is the warning log formatter From a637ffc8487d3da66900f4bf9e1ef2bcca7c082b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 24 Oct 2021 20:21:03 -0400 Subject: [PATCH 4/4] tests: skip test when in remote vscode container and also ensure the docker daemon config is valid --- tests/unit/logs.bats | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/logs.bats b/tests/unit/logs.bats index e1b640ff7..58af987c8 100644 --- a/tests/unit/logs.bats +++ b/tests/unit/logs.bats @@ -396,6 +396,14 @@ teardown() { } @test "(logs) logs:set max-size with alternate log-driver daemon " { + if [[ "$REMOTE_CONTAINERS" == "true" ]]; then + skip "skipping due non-existent docker service in remote dev container" + fi + + if [[ ! -f /etc/docker/daemon.json ]]; then + echo "{}" >/etc/docker/daemon.json + fi + driver="$(jq -r '."log-driver"' /etc/docker/daemon.json)" local TMP_FILE=$(mktemp "/tmp/dokku.me.XXXX")