feat: add vector integration

This currently is missing a few features:

- global sinks
- logrotation

But works fairly well otherwise.

plugins/common/docker.go

@@ -20,6 +20,31 @@ func ContainerIsRunning(containerID string) bool {
 	return strings.TrimSpace(string(b[:])) == "true"
 }
 
+// ContainerStart runs 'docker container start' against an existing container
+// whether that container is running or not
+func ContainerStart(containerID string) bool {
+	cmd := sh.Command(DockerBin(), "container", "start", containerID)
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	if err := cmd.Run(); err != nil {
+		return false
+	}
+
+	return true
+}
+
+// ContainerExists checks to see if a container exists
+func ContainerExists(containerID string) bool {
+	cmd := sh.Command(DockerBin(), "container", "inspect", containerID)
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	if err := cmd.Run(); err != nil {
+		return false
+	}
+
+	return true
+}
+
 // CopyFromImage copies a file from named image to destination
 func CopyFromImage(appName string, image string, source string, destination string) error {
 	if !VerifyImage(image) {
@@ -219,6 +244,11 @@ func DockerInspect(containerOrImageID, format string) (output string, err error)
 	return
 }
 
+// DockerInspect runs an inspect command with a given format against a container ID
+func DockerSignal(containerID, signal string) (err error) {
+	return sh.Command(DockerBin(), "kill", fmt.Sprintf("--signal=%s", signal), containerID).Run()
+}
+
 // IsImageHerokuishBased returns true if app image is based on herokuish
 func IsImageHerokuishBased(image string, appName string) bool {
 	output, err := DockerInspect(image, "{{range .Config.Env}}{{if eq . \"USER=herokuishuser\" }}{{println .}}{{end}}{{end}}")

plugins/common/log.go

@@ -1,6 +1,7 @@
 package common
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"strings"
@@ -93,6 +94,21 @@ func LogVerboseQuietContainerLogs(containerID string) {
 	}
 }
 
+// LogVerboseQuietContainerLogs is the verbose log formatter for container logs
+func LogVerboseQuietContainerLogsTail(containerID string) {
+	sc := NewShellCmdWithArgs(DockerBin(), "container", "logs", containerID, "--follow", "--tail", "10")
+	stdout, _ := sc.Command.StdoutPipe()
+	sc.Command.Start()
+
+	scanner := bufio.NewScanner(stdout)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		m := scanner.Text()
+		fmt.Println(m)
+	}
+	sc.Command.Wait()
+}
+
 // LogWarn is the warning log formatter
 func LogWarn(text string) {
 	fmt.Fprintln(os.Stderr, fmt.Sprintf(" !     %s", text))

plugins/logs/.gitignore

@@ -1,2 +1,7 @@
 /commands
 /subcommands/*
+/triggers/*
+/triggers
+/install
+/post-*
+/report

plugins/logs/Makefile

@@ -1,5 +1,6 @@
-SUBCOMMANDS = subcommands/failed
-BUILD = commands subcommands
+SUBCOMMANDS = subcommands/failed subcommands/report subcommands/set subcommands/vector-logs subcommands/vector-start subcommands/vector-stop
+TRIGGERS = triggers/install triggers/post-delete triggers/report
+BUILD = commands subcommands triggers
 PLUGIN_NAME = logs
 
 include ../../common.mk

plugins/logs/functions.go

@@ -0,0 +1,247 @@
+package logs
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/dokku/dokku/plugins/common"
+	"github.com/joncalhoun/qson"
+	"github.com/xo/dburl"
+)
+
+type vectorConfig struct {
+	Sources map[string]vectorSource `json:"sources"`
+	Sinks   map[string]vectorSink   `json:"sinks"`
+}
+
+type vectorSource struct {
+	Type          string   `json:"type"`
+	IncludeLabels []string `json:"include_labels,omitempty"`
+}
+
+type vectorSink map[string]interface{}
+
+const vectorContainerName = "vector"
+
+func init() {
+	sinks := []string{
+		"aws_cloudwatch_logs",
+		"aws_kinesis_firehose",
+		"aws_kinesis_streams",
+		"aws_s3", "aws_sqs",
+		"azure_monitor_logs",
+		"blackhole",
+		"clickhouse",
+		"console",
+		"datadog_logs",
+		"elasticsearch",
+		"file",
+		"gcp_cloud_storage",
+		"gcp_pubsub",
+		"gcp_stackdriver_logs",
+		"honeycomb",
+		"http",
+		"humio_logs",
+		"influxdb_logs",
+		"kafka",
+		"logdna",
+		"loki",
+		"nats",
+		"new_relic_logs",
+		"papertrail",
+		"pulsar",
+		"sematext_logs",
+		"socket",
+		"splunk_hec",
+		"vector",
+	}
+	for _, sink := range sinks {
+		dburl.Register(dburl.Scheme{
+			Driver:    sink,
+			Generator: genOpaqueWithMissingPath,
+			Proto:     0,
+			Opaque:    false,
+			Aliases:   []string{},
+			Override:  "",
+		})
+	}
+}
+
+func genOpaqueWithMissingPath(u *dburl.URL) (string, error) {
+	return u.Opaque + genQueryOptions(u.Query()), nil
+}
+
+func genQueryOptions(q url.Values) string {
+	if s := q.Encode(); s != "" {
+		return "?" + s
+	}
+	return ""
+}
+
+func killVectorContainer() error {
+	if !common.ContainerExists(vectorContainerName) {
+		return nil
+	}
+
+	if err := stopVectorContainer(); err != nil {
+		return err
+	}
+
+	time.Sleep(10 * time.Second)
+	if err := removeVectorContainer(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeVectorContainer() error {
+	if !common.ContainerExists(vectorContainerName) {
+		return nil
+	}
+
+	cmd := common.NewShellCmd(strings.Join([]string{
+		common.DockerBin(), "container", "rm", "-f", vectorContainerName}, " "))
+
+	return common.SuppressOutput(func() error {
+		if cmd.Execute() {
+			return nil
+		}
+
+		if common.ContainerExists(vectorContainerName) {
+			return errors.New("Unable to remove vector container")
+		}
+
+		return nil
+	})
+}
+
+func startVectorContainer(vectorImage string) error {
+	cmd := common.NewShellCmd(strings.Join([]string{
+		common.DockerBin(),
+		"container",
+		"run", "--detach", "--name", vectorContainerName, common.MustGetEnv("DOKKU_GLOBAL_RUN_ARGS"),
+		"--volume", "/var/lib/dokku/data/logs/vector.json:/etc/vector/vector.json",
+		"--volume", "/var/run/docker.sock:/var/run/docker.sock",
+		vectorImage,
+		"--config", "/etc/vector/vector.json"}, " "))
+
+	if !cmd.Execute() {
+		return errors.New("Unable to start vector container")
+	}
+
+	return nil
+}
+
+func stopVectorContainer() error {
+	if !common.ContainerExists(vectorContainerName) {
+		return nil
+	}
+
+	if !common.ContainerIsRunning(vectorContainerName) {
+		return nil
+	}
+
+	cmd := common.NewShellCmd(strings.Join([]string{
+		common.DockerBin(), "container", "stop", vectorContainerName}, " "))
+
+	return common.SuppressOutput(func() error {
+		if cmd.Execute() {
+			return nil
+		}
+
+		if common.ContainerIsRunning(vectorContainerName) {
+			return errors.New("Unable to stop vector container")
+		}
+
+		return nil
+	})
+}
+
+func valueToConfig(appName string, value string) (vectorSink, error) {
+	var data vectorSink
+	u, err := dburl.Parse(value)
+	if err != nil {
+		return data, err
+	}
+
+	if u.Query().Get("sinks") != "" {
+		return data, errors.New("Invalid option sinks")
+	}
+
+	t := fmt.Sprintf("type=%s", u.Scheme)
+	i := fmt.Sprintf("inputs[]=docker-source:%s", appName)
+	initialQuery := fmt.Sprintf("%s&%s", t, i)
+	query := u.RawQuery
+	if query == "" {
+		query = initialQuery
+	} else if strings.HasPrefix(query, "&") {
+		query = fmt.Sprintf("%s%s", initialQuery, query)
+	} else {
+		query = fmt.Sprintf("%s&%s", initialQuery, query)
+	}
+
+	b, err := qson.ToJSON(query)
+	if err != nil {
+		return data, err
+	}
+
+	if err := json.Unmarshal(b, &data); err != nil {
+		return data, err
+	}
+
+	return data, nil
+}
+
+func writeVectorConfig() error {
+	apps, err := common.DokkuApps()
+	if err != nil {
+		return err
+	}
+
+	data := vectorConfig{
+		Sources: map[string]vectorSource{},
+		Sinks:   map[string]vectorSink{},
+	}
+	for _, appName := range apps {
+		value := common.PropertyGet("logs", appName, "vector-sink")
+		if value == "" {
+			continue
+		}
+
+		sink, err := valueToConfig(appName, value)
+		if err != nil {
+			return err
+		}
+
+		data.Sources[fmt.Sprintf("docker-source:%s", appName)] = vectorSource{
+			Type:          "docker_logs",
+			IncludeLabels: []string{fmt.Sprintf("com.dokku.app-name=%s", appName)},
+		}
+
+		data.Sinks[fmt.Sprintf("docker-sink:%s", appName)] = sink
+	}
+
+	b, err := json.MarshalIndent(data, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	vectorConfig := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "logs", "vector.json")
+	if err := common.WriteSliceToFile(vectorConfig, []string{string(b)}); err != nil {
+		return err
+	}
+
+	if !common.ContainerExists(vectorContainerName) {
+		return nil
+	}
+
+	return common.SuppressOutput(func() error {
+		return common.DockerSignal(vectorContainerName, "HUP")
+	})
+}

plugins/logs/go.mod

@@ -3,10 +3,13 @@ module github.com/dokku/dokku/plugins/logs
 go 1.15
 
 require (
+	github.com/RaveNoX/go-jsonmerge v1.0.0
 	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27
 	github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
+	github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0
 	github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xo/dburl v0.1.0
 )
 
 replace github.com/dokku/dokku/plugins/common => ../common

plugins/logs/go.sum

@@ -1,10 +1,24 @@
+github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
+github.com/RaveNoX/go-jsonmerge v1.0.0 h1:2e0nqnadoGUP8rAvcA0hkQelZreVO5X3BHomT2XMrAk=
+github.com/RaveNoX/go-jsonmerge v1.0.0/go.mod h1:qYM/NA77LhO4h51JJM7Z+xBU3ovqrNIACZe+SkSNVFo=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
 github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0 h1:ct2XA1aDw8A07Dr8gtrrZgIgLKcZNAl2o9nn0WRMK4Y=
+github.com/joncalhoun/qson v0.0.0-20200422171543-84433dcd3da0/go.mod h1:DFXrEwSRX0p/aSvxE21319menCBFeQO0jXpRj7LEZUA=
+github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg=
 github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/xo/dburl v0.1.0 h1:WsW5ujxrapzhTflR4WoI+ZhynjE8T412ZWyoZtNNuJI=
+github.com/xo/dburl v0.1.0/go.mod h1:TM8VMBT+LWqC3MBOulZjb8FAthcvZq0t/qvDLwS6skU=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

plugins/logs/logs.go

@@ -6,6 +6,15 @@ import (
 	"github.com/dokku/dokku/plugins/common"
 )
 
+var (
+	// DefaultProperties is a map of all valid ps properties with corresponding default property values
+	DefaultProperties = map[string]string{
+		"vector-sink": "",
+	}
+)
+
+const VectorImage = "timberio/vector:0.11.X-debian"
+
 // GetFailedLogs outputs failed deploy logs for a given app
 func GetFailedLogs(appName string) error {
 	common.LogInfo2Quiet(fmt.Sprintf("%s failed deploy logs", appName))

plugins/logs/report.go

@@ -0,0 +1,25 @@
+package logs
+
+import (
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// ReportSingleApp is an internal function that displays the logs report for one or more apps
+func ReportSingleApp(appName, infoFlag string) error {
+	if err := common.VerifyAppName(appName); err != nil {
+		return err
+	}
+
+	flags := map[string]common.ReportFunc{
+		"--logs-vector-sink": reportVectorSink,
+	}
+
+	trimPrefix := false
+	uppercaseFirstCharacter := true
+	infoFlags := common.CollectReport(appName, infoFlag, flags)
+	return common.ReportSingleApp("logs", appName, infoFlag, infoFlags, trimPrefix, uppercaseFirstCharacter)
+}
+
+func reportVectorSink(appName string) string {
+	return common.PropertyGet("logs", appName, "vector-sink")
+}

plugins/logs/src/commands/commands.go

@@ -23,7 +23,12 @@ Additional commands:`
 
 	helpContent = `
     logs [-h] [-t|--tail] [-n|--num num] [-q|--quiet] [-p|--ps process] <app>, Display recent log output
-    logs:failed [<app>], Shows the last failed deploy logs
+    logs:failed [--all|<app>], Shows the last failed deploy logs
+    logs:report [<app>] [<flag>], Displays a logs report for one or more apps
+    logs:set <app> <key> <value>, Set or clear a logs property for an app
+    logs:vector-logs, Tail the logs of the vector container
+    logs:vector-start, Start the vector logging container
+    logs:vector-stop, Stop the vector logging container
 `
 )
 

plugins/logs/src/subcommands/subcommands.go

@@ -24,6 +24,34 @@ func main() {
 		args.Parse(os.Args[2:])
 		appName := args.Arg(0)
 		err = logs.CommandFailed(appName, *allApps)
+	case "report":
+		args := flag.NewFlagSet("logs:report", flag.ExitOnError)
+		osArgs, infoFlag, err := common.ParseReportArgs("logs", os.Args[2:])
+		if err == nil {
+			args.Parse(osArgs)
+			appName := args.Arg(0)
+			err = logs.CommandReport(appName, infoFlag)
+		}
+	case "set":
+		args := flag.NewFlagSet("logs:set", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		appName := args.Arg(0)
+		property := args.Arg(1)
+		value := args.Arg(2)
+		err = logs.CommandSet(appName, property, value)
+	case "vector-logs":
+		args := flag.NewFlagSet("logs:vector-logs", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		err = logs.CommandVectorLogs()
+	case "vector-start":
+		args := flag.NewFlagSet("logs:vector-start", flag.ExitOnError)
+		vectorImage := args.String("vector-image", logs.VectorImage, "--vector-image: the name of the docker image to run for vector")
+		args.Parse(os.Args[2:])
+		err = logs.CommandVectorStart(*vectorImage)
+	case "vector-stop":
+		args := flag.NewFlagSet("logs:vector-stop", flag.ExitOnError)
+		args.Parse(os.Args[2:])
+		err = logs.CommandVectorStop()
 	default:
 		common.LogFail(fmt.Sprintf("Invalid plugin subcommand call: %s", subcommand))
 	}

plugins/logs/src/triggers/triggers.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
+	"github.com/dokku/dokku/plugins/logs"
+)
+
+// main entrypoint to all triggers
+func main() {
+	parts := strings.Split(os.Args[0], "/")
+	trigger := parts[len(parts)-1]
+	flag.Parse()
+
+	var err error
+	switch trigger {
+	case "install":
+		err = logs.TriggerInstall()
+	case "post-delete":
+		appName := flag.Arg(0)
+		err = logs.TriggerPostDelete(appName)
+	case "report":
+		appName := flag.Arg(0)
+		err = logs.ReportSingleApp(appName, "")
+	default:
+		common.LogFail(fmt.Sprintf("Invalid plugin trigger call: %s", trigger))
+	}
+
+	if err != nil {
+		common.LogFail(err.Error())
+	}
+}

plugins/logs/subcommands.go

@@ -1,7 +1,9 @@
 package logs
 
 import (
+	"errors"
 	"fmt"
+	"path/filepath"
 	"strconv"
 
 	"github.com/dokku/dokku/plugins/common"
@@ -45,3 +47,80 @@ func CommandFailed(appName string, allApps bool) error {
 
 	return GetFailedLogs(appName)
 }
+
+// CommandReport displays a logs report for one or more apps
+func CommandReport(appName string, infoFlag string) error {
+	if len(appName) == 0 {
+		apps, err := common.DokkuApps()
+		if err != nil {
+			return err
+		}
+		for _, appName := range apps {
+			if err := ReportSingleApp(appName, infoFlag); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	return ReportSingleApp(appName, infoFlag)
+}
+
+// CommandSet sets or clears a logs property for an app
+func CommandSet(appName string, property string, value string) error {
+	if property == "vector-sink" && value != "" {
+		_, err := valueToConfig(appName, value)
+		if err != nil {
+			return err
+		}
+	}
+
+	common.CommandPropertySet("logs", appName, property, value, DefaultProperties)
+	if property == "vector-sink" {
+		common.LogVerboseQuiet(fmt.Sprintf("Writing updated vector config to %s", filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "logs", "vector.json")))
+		return writeVectorConfig()
+	}
+	return nil
+}
+
+// CommandVectorLogs tails the log output for the vector container
+func CommandVectorLogs() error {
+	if !common.ContainerExists(vectorContainerName) {
+		return errors.New("Vector container does not exist")
+	}
+
+	if !common.ContainerIsRunning(vectorContainerName) {
+		return errors.New("Vector container is not running")
+	}
+
+	common.LogInfo1Quiet("Tailing vector container logs")
+	common.LogVerboseQuietContainerLogsTail(vectorContainerName)
+
+	return nil
+}
+
+// CommandVectorStart starts a new vector container
+// or starts an existing one if it already exists
+func CommandVectorStart(vectorImage string) error {
+	common.LogInfo2("Starting vector container")
+	if !common.ContainerExists(vectorContainerName) {
+		return startVectorContainer(vectorImage)
+	}
+
+	if common.ContainerIsRunning(vectorContainerName) {
+		common.LogVerbose("Container already running")
+		return nil
+	}
+
+	if !common.ContainerStart(vectorContainerName) {
+		return errors.New("Unable to start vector container")
+	}
+
+	return nil
+}
+
+// CommandVectorStop stops and removes an existing vector container
+func CommandVectorStop() error {
+	common.LogInfo2Quiet("Stopping and removing vector container")
+	return killVectorContainer()
+}

plugins/logs/triggers.go

@@ -0,0 +1,41 @@
+package logs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// TriggerInstall initializes app restart policies
+func TriggerInstall() error {
+	if err := common.PropertySetup("logs"); err != nil {
+		return fmt.Errorf("Unable to install the logs plugin: %s", err.Error())
+	}
+
+	directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "logs")
+	if err := os.MkdirAll(directory, 0755); err != nil {
+		return err
+	}
+
+	if err := common.SetPermissions(directory, 0755); err != nil {
+		return err
+	}
+
+	logDirectory := filepath.Join(common.MustGetEnv("DOKKU_LOGS_DIR"), "apps")
+	if err := os.MkdirAll(logDirectory, 0755); err != nil {
+		return err
+	}
+
+	if err := common.SetPermissions(logDirectory, 0755); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// TriggerPostDelete destroys the network property for a given app container
+func TriggerPostDelete(appName string) error {
+	return common.PropertyDestroy("logs", appName)
+}