Merge pull request #6536 from dokku/vector-via-compose

Manage vector container via compose
This commit is contained in:
Jose Diaz-Gonzalez
2024-01-30 10:23:45 -05:00
committed by GitHub
7 changed files with 210 additions and 86 deletions

View File

@@ -2950,3 +2950,18 @@ main() {
main "$@" main "$@"
``` ```
### `vector-template-source`
- Description: Retrieves an alternative template for the vector compose config
- Invoked by: caddy-vhosts
- Arguments:
- Example:
```shell
#!/usr/bin/env bash
set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
# TODO
```

View File

@@ -33,6 +33,18 @@ func ContainerStart(containerID string) bool {
return true return true
} }
// ContainerRemove runs 'docker container remove' against an existing container
func ContainerRemove(containerID string) bool {
cmd := sh.Command(DockerBin(), "container", "remove", "-f", containerID)
cmd.Stdout = nil
cmd.Stderr = nil
if err := cmd.Run(); err != nil {
return false
}
return true
}
// ContainerExists checks to see if a container exists // ContainerExists checks to see if a container exists
func ContainerExists(containerID string) bool { func ContainerExists(containerID string) bool {
cmd := sh.Command(DockerBin(), "container", "inspect", containerID) cmd := sh.Command(DockerBin(), "container", "inspect", containerID)
@@ -259,6 +271,15 @@ func GetWorkingDir(appName string, image string) string {
return workDir return workDir
} }
func IsComposeInstalled() bool {
result, err := CallExecCommand(ExecCommandInput{
Command: DockerBin(),
Args: []string{"info", "--format", "{{range .ClientInfo.Plugins}}{{if eq .Name \"compose\"}}true{{end}}{{end}}')"},
CaptureOutput: true,
})
return err == nil && result.ExitCode == 0
}
// IsImageCnbBased returns true if app image is based on cnb // IsImageCnbBased returns true if app image is based on cnb
func IsImageCnbBased(image string) bool { func IsImageCnbBased(image string) bool {
if len(image) == 0 { if len(image) == 0 {

View File

@@ -5,11 +5,11 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"html/template"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
"github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/common"
"github.com/joncalhoun/qson" "github.com/joncalhoun/qson"
@@ -25,92 +25,164 @@ type vectorSource struct {
IncludeLabels []string `json:"include_labels,omitempty"` IncludeLabels []string `json:"include_labels,omitempty"`
} }
type vectorSink map[string]interface{} type vectorTemplateData struct {
DokkuLibRoot string
const vectorContainerName = "vector" DokkuLogsDir string
VectorImage string
func killVectorContainer() error {
if !common.ContainerExists(vectorContainerName) {
return nil
}
if err := stopVectorContainer(); err != nil {
return err
}
time.Sleep(10 * time.Second)
if err := removeVectorContainer(); err != nil {
return err
}
return nil
} }
func removeVectorContainer() error { type vectorSink map[string]interface{}
if !common.ContainerExists(vectorContainerName) {
return nil const vectorContainerName = "vector-vector-1"
const vectorOldContainerName = "vector"
func getComposeFile() ([]byte, error) {
result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "vector-template-source",
CaptureOutput: true,
})
if err == nil && result.ExitCode == 0 && strings.TrimSpace(result.Stdout) != "" {
contents, err := os.ReadFile(strings.TrimSpace(result.Stdout))
if err != nil {
return []byte{}, fmt.Errorf("Unable to read compose template: %s", err)
}
return contents, nil
} }
cmd := common.NewShellCmd(strings.Join([]string{ contents, err := templates.ReadFile("templates/compose.yml.tmpl")
common.DockerBin(), "container", "rm", "-f", vectorContainerName}, " ")) if err != nil {
return []byte{}, fmt.Errorf("Unable to read compose template: %s", err)
}
return common.SuppressOutput(func() error { return contents, nil
if cmd.Execute() {
return nil
}
if common.ContainerExists(vectorContainerName) {
return errors.New("Unable to remove vector container")
}
return nil
})
} }
func startVectorContainer(vectorImage string) error { func startVectorContainer(vectorImage string) error {
cmd := common.NewShellCmd(strings.Join([]string{ if !common.IsComposeInstalled() {
common.DockerBin(), return errors.New("Required docker compose plugin is not installed")
"container", }
"run", "--detach", "--name", vectorContainerName, common.MustGetEnv("DOKKU_GLOBAL_RUN_ARGS"),
"--restart", "unless-stopped",
"--volume", "/var/lib/dokku/data/logs:/etc/vector",
"--volume", "/var/run/docker.sock:/var/run/docker.sock",
"--volume", common.MustGetEnv("DOKKU_LOGS_HOST_DIR") + ":/var/logs/dokku/apps",
"--volume", common.MustGetEnv("DOKKU_LOGS_HOST_DIR") + "/apps:/var/log/dokku/apps",
vectorImage,
"--config", "/etc/vector/vector.json", "--watch-config"}, " "))
cmd.ShowOutput = false
if !cmd.Execute() { if common.ContainerExists(vectorOldContainerName) {
return errors.New("Unable to start vector container") return errors.New("Vector container %s already exists in old format, run 'dokku logs:vector-stop' once to remove it")
}
tmpFile, err := os.CreateTemp(os.TempDir(), "vector-compose-*.yml")
if err != nil {
return fmt.Errorf("Unable to create temporary file: %s", err)
}
defer os.Remove(tmpFile.Name())
contents, err := getComposeFile()
if err != nil {
return fmt.Errorf("Unable to read compose template: %s", err)
}
tmpl, err := template.New("compose.yml").Parse(string(contents))
if err != nil {
return fmt.Errorf("Unable to parse compose template: %s", err)
}
dokkuLibRoot := os.Getenv("DOKKU_LIB_HOST_ROOT")
if dokkuLibRoot == "" {
dokkuLibRoot = os.Getenv("DOKKU_LIB_ROOT")
}
dokkuLogsDir := os.Getenv("DOKKU_LOGS_HOST_DIR")
if dokkuLogsDir == "" {
dokkuLogsDir = os.Getenv("DOKKU_LOGS_DIR")
}
data := vectorTemplateData{
DokkuLibRoot: dokkuLibRoot,
DokkuLogsDir: dokkuLogsDir,
VectorImage: vectorImage,
}
if err := tmpl.Execute(tmpFile, data); err != nil {
return fmt.Errorf("Unable to execute compose template: %s", err)
}
result, err := common.CallExecCommand(common.ExecCommandInput{
Command: common.DockerBin(),
Args: []string{
"compose",
"--file", tmpFile.Name(),
"--project-name", "vector",
"up",
"--detach",
"--quiet-pull",
},
StreamStdio: true,
})
if err != nil || result.ExitCode != 0 {
return fmt.Errorf("Unable to start vector container: %s", result.Stderr)
} }
return nil return nil
} }
func stopVectorContainer() error { func stopVectorContainer() error {
if !common.ContainerExists(vectorContainerName) { if !common.IsComposeInstalled() {
return nil return errors.New("Required docker compose plugin is not installed")
} }
if !common.ContainerIsRunning(vectorContainerName) { if common.ContainerExists(vectorOldContainerName) {
return nil common.ContainerRemove(vectorOldContainerName)
} }
cmd := common.NewShellCmd(strings.Join([]string{ tmpFile, err := os.CreateTemp(os.TempDir(), "vector-compose-*.yml")
common.DockerBin(), "container", "stop", vectorContainerName}, " ")) if err != nil {
return fmt.Errorf("Unable to create temporary file: %s", err)
}
defer os.Remove(tmpFile.Name())
return common.SuppressOutput(func() error { contents, err := getComposeFile()
if cmd.Execute() { if err != nil {
return nil return fmt.Errorf("Unable to read compose template: %s", err)
} }
if common.ContainerIsRunning(vectorContainerName) { tmpl, err := template.New("compose.yml").Parse(string(contents))
return errors.New("Unable to stop vector container") if err != nil {
} return fmt.Errorf("Unable to parse compose template: %s", err)
}
return nil dokkuLibRoot := os.Getenv("DOKKU_LIB_HOST_ROOT")
if dokkuLibRoot == "" {
dokkuLibRoot = os.Getenv("DOKKU_LIB_ROOT")
}
dokkuLogsDir := os.Getenv("DOKKU_LOGS_HOST_DIR")
if dokkuLogsDir == "" {
dokkuLogsDir = os.Getenv("DOKKU_LOGS_DIR")
}
data := vectorTemplateData{
DokkuLibRoot: dokkuLibRoot,
DokkuLogsDir: dokkuLogsDir,
VectorImage: VectorImage,
}
if err := tmpl.Execute(tmpFile, data); err != nil {
return fmt.Errorf("Unable to execute compose template: %s", err)
}
result, err := common.CallExecCommand(common.ExecCommandInput{
Command: common.DockerBin(),
Args: []string{
"compose",
"--file", tmpFile.Name(),
"--project-name", "vector",
"down",
"--remove-orphans",
},
StreamStdio: true,
}) })
if err != nil || result.ExitCode != 0 {
return fmt.Errorf("Unable to stop vector container: %s", result.Stderr)
}
return nil
} }
func sinkValueToConfig(appName string, sinkValue string) (vectorSink, error) { func sinkValueToConfig(appName string, sinkValue string) (vectorSink, error) {
@@ -132,9 +204,7 @@ func sinkValueToConfig(appName string, sinkValue string) (vectorSink, error) {
u.Scheme = strings.ReplaceAll(u.Scheme, "-", "_") u.Scheme = strings.ReplaceAll(u.Scheme, "-", "_")
query := u.RawQuery query := u.RawQuery
if strings.HasPrefix(query, "&") { query = strings.TrimPrefix(query, "&")
query = strings.TrimPrefix(query, "&")
}
b, err := qson.ToJSON(query) b, err := qson.ToJSON(query)
if err != nil { if err != nil {

View File

@@ -1,6 +1,7 @@
package logs package logs
import ( import (
"embed"
"fmt" "fmt"
"github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/common"
@@ -30,6 +31,9 @@ const VectorImage = "timberio/vector:0.35.X-debian"
// VectorDefaultSink contains the default sink in use for vector log shipping // VectorDefaultSink contains the default sink in use for vector log shipping
const VectorDefaultSink = "blackhole://?print_interval_secs=1" const VectorDefaultSink = "blackhole://?print_interval_secs=1"
//go:embed templates/*
var templates embed.FS
// GetFailedLogs outputs failed deploy logs for a given app // GetFailedLogs outputs failed deploy logs for a given app
func GetFailedLogs(appName string) error { func GetFailedLogs(appName string) error {
common.LogInfo2Quiet(fmt.Sprintf("%s failed deploy logs", appName)) common.LogInfo2Quiet(fmt.Sprintf("%s failed deploy logs", appName))

View File

@@ -107,20 +107,9 @@ func CommandVectorStart(vectorImage string) error {
vectorImage = common.PropertyGetDefault("logs", "--global", "vector-image", VectorImage) vectorImage = common.PropertyGetDefault("logs", "--global", "vector-image", VectorImage)
} }
if common.ContainerExists(vectorContainerName) { common.LogVerbose("Starting vector container")
if common.ContainerIsRunning(vectorContainerName) { if err := startVectorContainer(vectorImage); err != nil {
common.LogVerbose("Vector container is running") return err
return nil
}
common.LogVerbose("Starting vector container")
if !common.ContainerStart(vectorContainerName) {
return errors.New("Unable to start vector container")
}
} else {
if err := startVectorContainer(vectorImage); err != nil {
return err
}
} }
common.LogVerbose("Waiting for 10 seconds") common.LogVerbose("Waiting for 10 seconds")
@@ -134,6 +123,6 @@ func CommandVectorStart(vectorImage string) error {
// CommandVectorStop stops and removes an existing vector container // CommandVectorStop stops and removes an existing vector container
func CommandVectorStop() error { func CommandVectorStop() error {
common.LogInfo2Quiet("Stopping and removing vector container") common.LogInfo2Quiet("Stopping and removing vector container")
return killVectorContainer() return stopVectorContainer()
} }

View File

@@ -0,0 +1,25 @@
---
version: "3.7"
services:
vector:
image: "{{ $.VectorImage }}"
command:
- "--config"
- "/etc/vector/vector.json"
- "--watch-config"
labels:
dokku: ""
org.label-schema.schema-version: "1.0"
org.label-schema.vendor: dokku
network_mode: bridge
restart: unless-stopped
volumes:
- "{{ $.DokkuLibRoot }}/data/logs:/etc/vector"
- "/var/run/docker.sock:/var/run/docker.sock:ro"
- "{{ $.DokkuLogsDir }}/apps:/var/log/dokku/apps"

View File

@@ -553,7 +553,7 @@ teardown() {
assert_success assert_success
assert_output_contains "Vector container is running" assert_output_contains "Vector container is running"
run /bin/bash -c "sudo docker inspect --format='{{.HostConfig.RestartPolicy.Name}}' vector" run /bin/bash -c "sudo docker inspect --format='{{.HostConfig.RestartPolicy.Name}}' vector-vector-1"
echo "output: $output" echo "output: $output"
echo "status: $status" echo "status: $status"
assert_success assert_success
@@ -586,7 +586,7 @@ teardown() {
assert_output_contains "vector:" 5 assert_output_contains "vector:" 5
assert_line_count 6 assert_line_count 6
run /bin/bash -c "docker stop vector" run /bin/bash -c "docker stop vector-vector-1"
echo "output: $output" echo "output: $output"
echo "status: $status" echo "status: $status"
assert_success assert_success
@@ -602,5 +602,5 @@ teardown() {
echo "output: $output" echo "output: $output"
echo "status: $status" echo "status: $status"
assert_success assert_success
assert_output_contains "Stopping and removing vector container" assert_output_contains "Stopping and removing vector container"
} }