From 5141360b48a747dc5a8d77d5ffe4536d945ff4ce Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 19 Feb 2021 23:33:57 -0500 Subject: [PATCH 1/2] refactor: move internal proxy functions to functions.go --- plugins/proxy/functions.go | 192 +++++++++++++++++++++++++++++++++++++ plugins/proxy/proxy.go | 185 ----------------------------------- 2 files changed, 192 insertions(+), 185 deletions(-) create mode 100644 plugins/proxy/functions.go diff --git a/plugins/proxy/functions.go b/plugins/proxy/functions.go new file mode 100644 index 000000000..bc12f32a9 --- /dev/null +++ b/plugins/proxy/functions.go @@ -0,0 +1,192 @@ +package proxy + +import ( + "errors" + "fmt" + "os" + "sort" + "strconv" + "strings" + + "github.com/dokku/dokku/plugins/common" + "github.com/dokku/dokku/plugins/config" + "github.com/ryanuber/columnize" +) + +func addProxyPorts(appName string, proxyPortMap []PortMap) error { + allPortMaps := getProxyPortMap(appName) + allPortMaps = append(allPortMaps, proxyPortMap...) + + return setProxyPorts(appName, allPortMaps) +} + +func filterAppProxyPorts(appName string, scheme string, hostPort int) []PortMap { + var filteredProxyMaps []PortMap + for _, portMap := range getProxyPortMap(appName) { + if portMap.Scheme == scheme && portMap.HostPort == hostPort { + filteredProxyMaps = append(filteredProxyMaps, portMap) + } + } + + return filteredProxyMaps +} + +func getAppProxyType(appName string) string { + return config.GetWithDefault(appName, "DOKKU_APP_PROXY_TYPE", "nginx") +} + +func getProxyPortMap(appName string) []PortMap { + value := config.GetWithDefault(appName, "DOKKU_PROXY_PORT_MAP", "") + portMaps, _ := parseProxyPortMapString(value) + return portMaps +} + +func inRange(value int, min int, max int) bool { + return min < value && value < max +} + +func listAppProxyPorts(appName string) error { + proxyPortMap := getProxyPortMap(appName) + + if len(proxyPortMap) == 0 { + return errors.New("No port mappings configured for app") + } + + var lines []string + if os.Getenv("DOKKU_QUIET_OUTPUT") == "" { + lines = append(lines, "-----> scheme:host port:container port") + } + + for _, portMap := range proxyPortMap { + lines = append(lines, portMap.String()) + } + + sort.Strings(lines) + common.LogInfo1Quiet(fmt.Sprintf("Port mappings for %s", appName)) + config := columnize.DefaultConfig() + config.Delim = ":" + config.Prefix = " " + config.Empty = "" + fmt.Println(columnize.Format(lines, config)) + return nil +} + +func parseProxyPortMapString(stringPortMap string) ([]PortMap, error) { + var proxyPortMap []PortMap + + for _, v := range strings.Split(strings.TrimSpace(stringPortMap), " ") { + parts := strings.SplitN(v, ":", 3) + if len(parts) == 1 { + hostPort, err := strconv.Atoi(v) + if err != nil { + return proxyPortMap, fmt.Errorf("Invalid port map %s [err=%s]", v, err.Error()) + } + + if !inRange(hostPort, 0, 65536) { + return proxyPortMap, fmt.Errorf("Invalid port map %s [hostPort=%d]", v, hostPort) + } + + proxyPortMap = append(proxyPortMap, PortMap{ + HostPort: hostPort, + Scheme: "__internal__", + }) + continue + } + + if len(parts) != 3 { + return proxyPortMap, fmt.Errorf("Invalid port map %s [len=%d]", v, len(parts)) + } + + hostPort, err := strconv.Atoi(parts[1]) + if err != nil { + return proxyPortMap, fmt.Errorf("Invalid port map %s [err=%s]", v, err.Error()) + } + + containerPort, err := strconv.Atoi(parts[2]) + if err != nil { + return proxyPortMap, fmt.Errorf("Invalid port map %s [err=%s]", v, err.Error()) + } + + if !inRange(hostPort, 0, 65536) { + return proxyPortMap, fmt.Errorf("Invalid port map %s [hostPort=%d]", v, hostPort) + } + + if !inRange(containerPort, 0, 65536) { + return proxyPortMap, fmt.Errorf("Invalid port map %s [containerPort=%d]", v, containerPort) + } + + proxyPortMap = append(proxyPortMap, PortMap{ + ContainerPort: containerPort, + HostPort: hostPort, + Scheme: parts[0], + }) + } + + return uniqueProxyPortMap(proxyPortMap), nil +} + +func removeProxyPorts(appName string, proxyPortMap []PortMap) error { + toRemove := map[string]bool{} + toRemoveByPort := map[int]bool{} + + for _, portMap := range proxyPortMap { + if portMap.AllowsPersistence() { + toRemoveByPort[portMap.HostPort] = true + continue + } + toRemove[portMap.String()] = true + } + + var toSet []PortMap + for _, portMap := range getProxyPortMap(appName) { + if toRemove[portMap.String()] { + continue + } + + if toRemoveByPort[portMap.HostPort] { + continue + } + + toSet = append(toSet, portMap) + } + + if len(toSet) == 0 { + keys := []string{"DOKKU_PROXY_PORT_MAP"} + return config.UnsetMany(appName, keys, false) + } + + return setProxyPorts(appName, toSet) +} + +func setProxyPorts(appName string, proxyPortMap []PortMap) error { + var value []string + for _, portMap := range uniqueProxyPortMap(proxyPortMap) { + if portMap.AllowsPersistence() { + continue + } + + value = append(value, portMap.String()) + } + + sort.Strings(value) + entries := map[string]string{ + "DOKKU_PROXY_PORT_MAP": strings.Join(value, " "), + } + return config.SetMany(appName, entries, false) +} + +func uniqueProxyPortMap(proxyPortMap []PortMap) []PortMap { + var unique []PortMap + existingPortMaps := map[string]bool{} + + for _, portMap := range proxyPortMap { + if existingPortMaps[portMap.String()] { + continue + } + + existingPortMaps[portMap.String()] = true + unique = append(unique, portMap) + } + + return unique +} diff --git a/plugins/proxy/proxy.go b/plugins/proxy/proxy.go index cec4abce5..007c63570 100644 --- a/plugins/proxy/proxy.go +++ b/plugins/proxy/proxy.go @@ -1,16 +1,9 @@ package proxy import ( - "errors" "fmt" - "os" - "sort" - "strconv" - "strings" - "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" - "github.com/ryanuber/columnize" ) // PortMap is a struct that contains a scheme:host-port:container-port mapping @@ -29,10 +22,6 @@ func (p PortMap) AllowsPersistence() bool { return p.Scheme == "__internal__" } -func inRange(value int, min int, max int) bool { - return min < value && value < max -} - // IsAppProxyEnabled returns true if proxy is enabled; otherwise return false func IsAppProxyEnabled(appName string) bool { proxyEnabled := true @@ -42,177 +31,3 @@ func IsAppProxyEnabled(appName string) bool { } return proxyEnabled } - -func addProxyPorts(appName string, proxyPortMap []PortMap) error { - allPortMaps := getProxyPortMap(appName) - allPortMaps = append(allPortMaps, proxyPortMap...) - - return setProxyPorts(appName, allPortMaps) -} - -func filterAppProxyPorts(appName string, scheme string, hostPort int) []PortMap { - var filteredProxyMaps []PortMap - for _, portMap := range getProxyPortMap(appName) { - if portMap.Scheme == scheme && portMap.HostPort == hostPort { - filteredProxyMaps = append(filteredProxyMaps, portMap) - } - } - - return filteredProxyMaps -} - -func getAppProxyType(appName string) string { - return config.GetWithDefault(appName, "DOKKU_APP_PROXY_TYPE", "nginx") -} - -func getProxyPortMap(appName string) []PortMap { - value := config.GetWithDefault(appName, "DOKKU_PROXY_PORT_MAP", "") - portMaps, _ := parseProxyPortMapString(value) - return portMaps -} - -func listAppProxyPorts(appName string) error { - proxyPortMap := getProxyPortMap(appName) - - if len(proxyPortMap) == 0 { - return errors.New("No port mappings configured for app") - } - - var lines []string - if os.Getenv("DOKKU_QUIET_OUTPUT") == "" { - lines = append(lines, "-----> scheme:host port:container port") - } - - for _, portMap := range proxyPortMap { - lines = append(lines, portMap.String()) - } - - sort.Strings(lines) - common.LogInfo1Quiet(fmt.Sprintf("Port mappings for %s", appName)) - config := columnize.DefaultConfig() - config.Delim = ":" - config.Prefix = " " - config.Empty = "" - fmt.Println(columnize.Format(lines, config)) - return nil -} - -func setProxyPorts(appName string, proxyPortMap []PortMap) error { - var value []string - for _, portMap := range uniqueProxyPortMap(proxyPortMap) { - if portMap.AllowsPersistence() { - continue - } - - value = append(value, portMap.String()) - } - - sort.Strings(value) - entries := map[string]string{ - "DOKKU_PROXY_PORT_MAP": strings.Join(value, " "), - } - return config.SetMany(appName, entries, false) -} - -func removeProxyPorts(appName string, proxyPortMap []PortMap) error { - toRemove := map[string]bool{} - toRemoveByPort := map[int]bool{} - - for _, portMap := range proxyPortMap { - if portMap.AllowsPersistence() { - toRemoveByPort[portMap.HostPort] = true - continue - } - toRemove[portMap.String()] = true - } - - var toSet []PortMap - for _, portMap := range getProxyPortMap(appName) { - if toRemove[portMap.String()] { - continue - } - - if toRemoveByPort[portMap.HostPort] { - continue - } - - toSet = append(toSet, portMap) - } - - if len(toSet) == 0 { - keys := []string{"DOKKU_PROXY_PORT_MAP"} - return config.UnsetMany(appName, keys, false) - } - - return setProxyPorts(appName, toSet) -} - -func parseProxyPortMapString(stringPortMap string) ([]PortMap, error) { - var proxyPortMap []PortMap - - for _, v := range strings.Split(strings.TrimSpace(stringPortMap), " ") { - parts := strings.SplitN(v, ":", 3) - if len(parts) == 1 { - hostPort, err := strconv.Atoi(v) - if err != nil { - return proxyPortMap, fmt.Errorf("Invalid port map %s [err=%s]", v, err.Error()) - } - - if !inRange(hostPort, 0, 65536) { - return proxyPortMap, fmt.Errorf("Invalid port map %s [hostPort=%d]", v, hostPort) - } - - proxyPortMap = append(proxyPortMap, PortMap{ - HostPort: hostPort, - Scheme: "__internal__", - }) - continue - } - - if len(parts) != 3 { - return proxyPortMap, fmt.Errorf("Invalid port map %s [len=%d]", v, len(parts)) - } - - hostPort, err := strconv.Atoi(parts[1]) - if err != nil { - return proxyPortMap, fmt.Errorf("Invalid port map %s [err=%s]", v, err.Error()) - } - - containerPort, err := strconv.Atoi(parts[2]) - if err != nil { - return proxyPortMap, fmt.Errorf("Invalid port map %s [err=%s]", v, err.Error()) - } - - if !inRange(hostPort, 0, 65536) { - return proxyPortMap, fmt.Errorf("Invalid port map %s [hostPort=%d]", v, hostPort) - } - - if !inRange(containerPort, 0, 65536) { - return proxyPortMap, fmt.Errorf("Invalid port map %s [containerPort=%d]", v, containerPort) - } - - proxyPortMap = append(proxyPortMap, PortMap{ - ContainerPort: containerPort, - HostPort: hostPort, - Scheme: parts[0], - }) - } - - return uniqueProxyPortMap(proxyPortMap), nil -} - -func uniqueProxyPortMap(proxyPortMap []PortMap) []PortMap { - var unique []PortMap - existingPortMaps := map[string]bool{} - - for _, portMap := range proxyPortMap { - if existingPortMaps[portMap.String()] { - continue - } - - existingPortMaps[portMap.String()] = true - unique = append(unique, portMap) - } - - return unique -} From 04a96125cc3e7b5fc55904f5557f38400ce0b0ba Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Fri, 19 Feb 2021 23:44:41 -0500 Subject: [PATCH 2/2] feat: add parallelism to certain proxy commands Closes #4392 --- docs/appendices/0.24.0-migration-guide.md | 5 ++ docs/networking/proxy-management.md | 34 ++++++++++---- plugins/proxy/proxy.go | 45 ++++++++++++++++++ plugins/proxy/src/subcommands/subcommands.go | 12 +++-- plugins/proxy/subcommands.go | 49 +++++++------------- 5 files changed, 103 insertions(+), 42 deletions(-) create mode 100644 docs/appendices/0.24.0-migration-guide.md diff --git a/docs/appendices/0.24.0-migration-guide.md b/docs/appendices/0.24.0-migration-guide.md new file mode 100644 index 000000000..d4fa0a7ea --- /dev/null +++ b/docs/appendices/0.24.0-migration-guide.md @@ -0,0 +1,5 @@ +# 0.23.0 Migration Guide + +## Changes + +- The commands `proxy:enable`, `proxy:disable` and `proxy:build-config` now support the `--all` flag in addition to general parallelism. diff --git a/docs/networking/proxy-management.md b/docs/networking/proxy-management.md index 1a8b24ae8..d7f5fe21d 100644 --- a/docs/networking/proxy-management.md +++ b/docs/networking/proxy-management.md @@ -3,11 +3,11 @@ > New as of 0.5.0, Enhanced in 0.6.0 ``` -proxy:build-config # (Re)builds config for given app -proxy:disable # Disable proxy for app -proxy:enable # Enable proxy for app -proxy:report [] [] # Displays a proxy report for one or more apps -proxy:set # Set proxy type for app +proxy:build-config [--parallel count] [--all|] # (Re)builds config for given app +proxy:disable [--parallel count] [--all|] # Disable proxy for app +proxy:enable [--parallel count] [--all|] # Enable proxy for app +proxy:report [] [] # Displays a proxy report for one or more apps +proxy:set # Set proxy type for app ``` In Dokku 0.5.0, port proxying was decoupled from the `nginx-vhosts` plugin into the proxy plugin. Dokku 0.6.0 introduced the ability to map host ports to specific container ports. In the future this will allow other proxy software - such as HAProxy or Caddy - to be used in place of nginx. @@ -22,11 +22,23 @@ In certain cases, your app proxy configs may drift from the correct config for y dokku proxy:build-config node-js-app ``` -### Container network interface binding +All apps may have their proxy config rebuilt by using the `--all` flag. -> Changed as of 0.11.0 +```shell +dokku proxy:build-config --all +``` -From Dokku versions `0.5.0` until `0.11.0`, enabling or disabling an application's proxy would **also** control whether or not the application was bound to all interfaces - e.g. `0.0.0.0`. As of `0.11.0`, this is now controlled by the network plugin. Please see the [network documentation](/docs/networking/network.md#container-network-interface-binding) for more information. +By default, rebuilding proxy configs for all apps happens serially. The parallelism may be controlled by the `--parallel` flag. + +```shell +dokku proxy:build-config --all --parallel 2 +``` + +Finally, the number of parallel workers may be automatically set to the number of CPUs available by setting the `--parallel` flag to `-1` + +```shell +dokku proxy:build-config --all --parallel -1 +``` ### Displaying proxy reports for an app @@ -94,3 +106,9 @@ dokku proxy:set node-js-app nginx ### Proxy port mapping See the [port management documentation](/docs/networking/port-management.md) for more information on how port mappings are managed for an application. + +### Container network interface binding + +> Changed as of 0.11.0 + +From Dokku versions `0.5.0` until `0.11.0`, enabling or disabling an application's proxy would **also** control whether or not the application was bound to all interfaces - e.g. `0.0.0.0`. As of `0.11.0`, this is now controlled by the network plugin. Please see the [network documentation](/docs/networking/network.md#container-network-interface-binding) for more information. diff --git a/plugins/proxy/proxy.go b/plugins/proxy/proxy.go index 007c63570..817602cea 100644 --- a/plugins/proxy/proxy.go +++ b/plugins/proxy/proxy.go @@ -3,9 +3,14 @@ package proxy import ( "fmt" + "github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/config" ) +// RunInSerial is the default value for whether to run a command in parallel or not +// and defaults to -1 (false) +const RunInSerial = 0 + // PortMap is a struct that contains a scheme:host-port:container-port mapping type PortMap struct { ContainerPort int @@ -22,6 +27,46 @@ func (p PortMap) AllowsPersistence() bool { return p.Scheme == "__internal__" } +// BuildConfig rebuilds the proxy config for the specified app +func BuildConfig(appName string) error { + return common.PlugnTrigger("proxy-build-config", []string{appName}...) +} + +// Disable disables proxy implementations for the specified app +func Disable(appName string) error { + if !IsAppProxyEnabled(appName) { + common.LogInfo1("Proxy is already disable for app") + return nil + } + + common.LogInfo1("Disabling proxy for app") + entries := map[string]string{ + "DOKKU_DISABLE_PROXY": "1", + } + + if err := config.SetMany(appName, entries, false); err != nil { + return err + } + + return common.PlugnTrigger("proxy-disable", []string{appName}...) +} + +// Enable enables proxy implementations for the specified app +func Enable(appName string) error { + if IsAppProxyEnabled(appName) { + common.LogInfo1("Proxy is already enabled for app") + return nil + } + + common.LogInfo1("Enabling proxy for app") + keys := []string{"DOKKU_DISABLE_PROXY"} + if err := config.UnsetMany(appName, keys, false); err != nil { + return err + } + + return common.PlugnTrigger("proxy-enable", []string{appName}...) +} + // IsAppProxyEnabled returns true if proxy is enabled; otherwise return false func IsAppProxyEnabled(appName string) bool { proxyEnabled := true diff --git a/plugins/proxy/src/subcommands/subcommands.go b/plugins/proxy/src/subcommands/subcommands.go index 1fb7ad857..25405964a 100644 --- a/plugins/proxy/src/subcommands/subcommands.go +++ b/plugins/proxy/src/subcommands/subcommands.go @@ -20,19 +20,25 @@ func main() { switch subcommand { case "build-config": args := flag.NewFlagSet("proxy:build-config", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: build-config for all apps") + parallelCount := args.Int("parallel", proxy.RunInSerial, "--parallel: number of apps to build-config for in parallel, -1 to match cpu count") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = proxy.CommandBuildConfig(appName) + err = proxy.CommandBuildConfig(appName, *allApps, *parallelCount) case "disable": args := flag.NewFlagSet("proxy:disable", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: disable proxy for all apps") + parallelCount := args.Int("parallel", proxy.RunInSerial, "--parallel: number of apps to disable proxy for in parallel, -1 to match cpu count") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = proxy.CommandDisable(appName) + err = proxy.CommandDisable(appName, *allApps, *parallelCount) case "enable": args := flag.NewFlagSet("proxy:enable", flag.ExitOnError) + allApps := args.Bool("all", false, "--all: enable proxy for all apps") + parallelCount := args.Int("parallel", proxy.RunInSerial, "--parallel: number of apps to enable proxy for in parallel, -1 to match cpu count") args.Parse(os.Args[2:]) appName := args.Arg(0) - err = proxy.CommandEnable(appName) + err = proxy.CommandEnable(appName, *allApps, *parallelCount) case "ports": args := flag.NewFlagSet("proxy:ports", flag.ExitOnError) args.Parse(os.Args[2:]) diff --git a/plugins/proxy/subcommands.go b/plugins/proxy/subcommands.go index 8652444c7..44a035f26 100644 --- a/plugins/proxy/subcommands.go +++ b/plugins/proxy/subcommands.go @@ -9,55 +9,42 @@ import ( ) // CommandBuildConfig rebuilds config for a given app -func CommandBuildConfig(appName string) error { +func CommandBuildConfig(appName string, allApps bool, parallelCount int) error { + if allApps { + return common.RunCommandAgainstAllApps(BuildConfig, "build-config", parallelCount) + } + if err := common.VerifyAppName(appName); err != nil { return err } - return common.PlugnTrigger("proxy-build-config", []string{appName}...) + return BuildConfig(appName) } // CommandDisable disables the proxy for app via command line -func CommandDisable(appName string) error { +func CommandDisable(appName string, allApps bool, parallelCount int) error { + if allApps { + return common.RunCommandAgainstAllApps(Disable, "disable", parallelCount) + } + if err := common.VerifyAppName(appName); err != nil { return err } - if !IsAppProxyEnabled(appName) { - common.LogInfo1("Proxy is already disable for app") - return nil - } - - common.LogInfo1("Disabling proxy for app") - entries := map[string]string{ - "DOKKU_DISABLE_PROXY": "1", - } - - if err := config.SetMany(appName, entries, false); err != nil { - return err - } - - return common.PlugnTrigger("proxy-disable", []string{appName}...) + return Disable(appName) } // CommandEnable enables the proxy for app via command line -func CommandEnable(appName string) error { +func CommandEnable(appName string, allApps bool, parallelCount int) error { + if allApps { + return common.RunCommandAgainstAllApps(Enable, "enable", parallelCount) + } + if err := common.VerifyAppName(appName); err != nil { return err } - if IsAppProxyEnabled(appName) { - common.LogInfo1("Proxy is already enabled for app") - return nil - } - - common.LogInfo1("Enabling proxy for app") - keys := []string{"DOKKU_DISABLE_PROXY"} - if err := config.UnsetMany(appName, keys, false); err != nil { - return err - } - - return common.PlugnTrigger("proxy-enable", []string{appName}...) + return Enable(appName) } // CommandPorts is a cmd wrapper to list proxy port mappings for an app