From 401683a3059350dedeabd7e74942ed68f113637b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 1 Aug 2021 17:47:04 -0400 Subject: [PATCH] refactor: write DOKKU_SCALE contents to scale property - remove extraction of DOKKU_SCALE - migrate to property system - rewrite internals to use a FormationSlice instead of a map --- plugins/common/properties.go | 29 ++--- plugins/common/src/prop/prop.go | 10 ++ plugins/ps/functions.go | 211 ++++++++------------------------ plugins/ps/subcommands.go | 14 +-- plugins/ps/triggers.go | 31 ++++- tests/unit/ps-general.bats | 2 +- 6 files changed, 105 insertions(+), 192 deletions(-) diff --git a/plugins/common/properties.go b/plugins/common/properties.go index 9b9f9fec7..8b7b58b41 100644 --- a/plugins/common/properties.go +++ b/plugins/common/properties.go @@ -173,6 +173,15 @@ func PropertyListAdd(pluginName string, appName string, property string, value s lines = append(lines, value) } + return PropertyListWrite(pluginName, appName, property, lines) +} + +// PropertyListWrite completely rewrites a list property +func PropertyListWrite(pluginName string, appName string, property string, values []string) error { + if err := propertyTouch(pluginName, appName, property); err != nil { + return err + } + propertyPath := getPropertyPath(pluginName, appName, property) file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600) if err != nil { @@ -180,7 +189,7 @@ func PropertyListAdd(pluginName string, appName string, property string, value s } w := bufio.NewWriter(file) - for _, line := range lines { + for _, line := range values { fmt.Fprintln(w, line) } if err = w.Flush(); err != nil { @@ -389,23 +398,7 @@ func PropertyListSet(pluginName string, appName string, property string, value s } } - propertyPath := getPropertyPath(pluginName, appName, property) - file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return err - } - - w := bufio.NewWriter(file) - for _, line := range lines { - fmt.Fprintln(w, line) - } - if err = w.Flush(); err != nil { - return fmt.Errorf("Unable to write %s config value %s.%s: %s", pluginName, appName, property, err.Error()) - } - - file.Chmod(0600) - SetPermissions(propertyPath, 0600) - return nil + return PropertyListWrite(pluginName, appName, property, lines) } // propertyTouch ensures a given application property file exists diff --git a/plugins/common/src/prop/prop.go b/plugins/common/src/prop/prop.go index c6ce2d0cf..9c06bb2bb 100644 --- a/plugins/common/src/prop/prop.go +++ b/plugins/common/src/prop/prop.go @@ -159,6 +159,16 @@ func main() { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } + case "lwrite": + _, args := common.ShiftString(flag.Args()) + appName, args := common.ShiftString(args) + property, args := common.ShiftString(args) + _, values := common.ShiftString(args) + err := common.PropertyListWrite(pluginName, appName, property, values) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } case "rpush": appName := flag.Arg(2) property := flag.Arg(3) diff --git a/plugins/ps/functions.go b/plugins/ps/functions.go index b318c86df..134f97f32 100644 --- a/plugins/ps/functions.go +++ b/plugins/ps/functions.go @@ -47,64 +47,6 @@ func extractProcfile(appName, image string) error { return nil } -func extractOrGenerateScalefile(appName string, image string) error { - destination := getScalefilePath(appName) - extracted := getScalefileExtractedPath(appName) - previouslyExtracted := common.FileExists(extracted) - - if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", extracted); err != nil { - if previouslyExtracted { - os.Remove(destination) - } - os.Remove(extracted) - } else if err := common.CopyFile(extracted, destination); err != nil { - return err - } - - if common.FileExists(destination) { - common.LogInfo1Quiet("DOKKU_SCALE file exists") - return updateScalefile(appName, false, make(map[string]int)) - } - - common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...") - if err := updateScalefile(appName, false, make(map[string]int)); err != nil { - common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) - return err - } - return nil -} - -func generateScalefile(appName string) error { - destination := getScalefilePath(appName) - procfilePath := getProcfilePath(appName) - content := []string{"web=1"} - if !common.FileExists(procfilePath) { - return common.WriteSliceToFile(destination, content) - } - - lines, err := common.FileToSlice(procfilePath) - if err != nil { - return common.WriteSliceToFile(destination, content) - } - - content = []string{} - for _, line := range lines { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - parts := strings.Split(line, ":") - count := 0 - if parts[0] == "web" { - count = 1 - } - - content = append(content, fmt.Sprintf("%s=%d", parts[0], count)) - } - - return common.WriteSliceToFile(destination, content) -} - func getProcessStatus(appName string) map[string]string { statuses := make(map[string]string) containerFiles := common.ListFilesWithPrefix(common.AppRoot(appName), "CONTAINER.") @@ -183,19 +125,6 @@ func getRunningState(appName string) string { return strings.Split(strings.TrimSpace(string(b[:])), " ")[1] } -func getScalefilePath(appName string) string { - return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE") -} - -func getScalefileExtractedPath(appName string) string { - return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") -} - -func hasScaleFile(appName string) bool { - scalefilePath := getScalefilePath(appName) - return common.FileExists(scalefilePath) -} - func hasProcfile(appName string) bool { procfilePath := getProcfilePath(appName) return common.FileExists(procfilePath) @@ -220,25 +149,28 @@ func isValidRestartPolicy(policy string) bool { return strings.HasPrefix(policy, "on-failure:") } -func parseProcessTuples(processTuples []string) (map[string]int, error) { - scale := make(map[string]int) +func parseProcessTuples(processTuples []string) (FormationSlice, error) { + formations := FormationSlice{} for _, processTuple := range processTuples { s := strings.Split(processTuple, "=") if len(s) == 1 { - return scale, fmt.Errorf("Missing count for process type %s", processTuple) + return formations, fmt.Errorf("Missing count for process type %s", processTuple) } processType := s[0] - count, err := strconv.Atoi(s[1]) + quantity, err := strconv.Atoi(s[1]) if err != nil { - return scale, fmt.Errorf("Invalid count for process type %s", s[0]) + return formations, fmt.Errorf("Invalid count for process type %s", s[0]) } - scale[processType] = count + formations = append(formations, &Formation{ + ProcessType: processType, + Quantity: quantity, + }) } - return scale, nil + return formations, nil } func processesInProcfile(procfilePath string) (map[string]bool, error) { @@ -266,63 +198,14 @@ func processesInProcfile(procfilePath string) (map[string]bool, error) { return processes, nil } -func readScaleFile(appName string) (map[string]int, error) { - scale := make(map[string]int) - if !hasScaleFile(appName) { - return scale, errors.New("No DOKKU_SCALE file found") - } - - scalefilePath := getScalefilePath(appName) - lines, err := common.FileToSlice(scalefilePath) - if err != nil { - return scale, err - } - - for i, line := range lines { - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - s := strings.Split(line, "=") - if len(s) != 2 { - common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i)) - continue - } - - processType := s[0] - count, err := strconv.Atoi(s[1]) - if err != nil { - common.LogWarn(fmt.Sprintf("Invalid count on line %d, skipping", i)) - continue - } - - scale[processType] = count - } - - if len(scale) == 0 { - common.LogWarn("No valid entries found in scale file, defaulting to web=1") - scale["web"] = 1 - } - - return scale, nil -} - func getFormations(appName string) (FormationSlice, error) { - scale := FormationSlice{} - data, err := readScaleFile(appName) + formations := FormationSlice{} + processTuples, err := common.PropertyListGet("ps", appName, "scale") if err != nil { - return scale, err + return formations, err } - for processType, quantity := range data { - f := Formation{ - ProcessType: processType, - Quantity: quantity, - } - scale = append(scale, &f) - } - - return scale, nil + return parseProcessTuples(processTuples) } func removeProcfile(appName string) error { @@ -382,13 +265,12 @@ func scaleReport(appName string) error { } func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error { - scale, err := parseProcessTuples(processTuples) + formations, err := parseProcessTuples(processTuples) if err != nil { return err } - common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) - if err := updateScalefile(appName, clearExisting, scale); err != nil { + if err := updateScale(appName, clearExisting, formations); err != nil { return err } @@ -408,17 +290,15 @@ func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) } -func updateScalefile(appName string, clearExisting bool, scaleUpdates map[string]int) error { - if !hasScaleFile(appName) { - if err := generateScalefile(appName); err != nil { +func updateScale(appName string, clearExisting bool, formationUpdates FormationSlice) error { + formations := FormationSlice{} + if !clearExisting { + processTuples, err := common.PropertyListGet("ps", appName, "scale") + if err != nil { return err } - } - scale := make(map[string]int) - if !clearExisting { - var err error - scale, err = readScaleFile(appName) + formations, err = parseProcessTuples(processTuples) if err != nil { return err } @@ -435,30 +315,43 @@ func updateScalefile(appName string, clearExisting bool, scaleUpdates map[string } } - for processType, count := range scaleUpdates { - if procfileExists && !validProcessTypes[processType] && count != 0 { - return fmt.Errorf("%s is not a valid process name to scale up", processType) + foundProcessTypes := map[string]bool{} + updatedFormation := FormationSlice{} + for _, formation := range formationUpdates { + if procfileExists && !validProcessTypes[formation.ProcessType] && formation.Quantity != 0 { + return fmt.Errorf("%s is not a valid process name to scale up", formation.ProcessType) } - scale[processType] = count + + foundProcessTypes[formation.ProcessType] = true + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: formation.ProcessType, + Quantity: formation.Quantity, + }) + } + + for _, formation := range formations { + foundProcessTypes[formation.ProcessType] = true + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: formation.ProcessType, + Quantity: formation.Quantity, + }) } for processType := range validProcessTypes { - count, ok := scale[processType] - if !ok { - count = 0 + if foundProcessTypes[processType] { + continue } - scale[processType] = count + + updatedFormation = append(updatedFormation, &Formation{ + ProcessType: processType, + Quantity: 0, + }) } - content := []string{} - for processType, count := range scale { - content = append(content, fmt.Sprintf("%s=%d", processType, count)) + values := []string{} + for _, formation := range formations { + values = append(values, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity)) } - scalefilePath := getScalefilePath(appName) - if err := common.WriteSliceToFile(scalefilePath, content); err != nil { - return err - } - - return nil + return common.PropertyListWrite("ps", appName, "scale", values) } diff --git a/plugins/ps/subcommands.go b/plugins/ps/subcommands.go index 2fb208ba5..1ba4703d2 100644 --- a/plugins/ps/subcommands.go +++ b/plugins/ps/subcommands.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" dockeroptions "github.com/dokku/dokku/plugins/docker-options" @@ -120,24 +121,15 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error return err } - procfilePath := getProcfilePath(appName) - if !hasScaleFile(appName) || common.FileExists(procfilePath) { - update := func() error { - return updateScalefile(appName, false, make(map[string]int)) - } - if err := common.SuppressOutput(update); err != nil { - return err - } - } - if len(processTuples) == 0 { return scaleReport(appName) } if !canScaleApp(appName) { - return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) + return fmt.Errorf("App %s contains an app.json file with a formations key and cannot be manually scaled", appName) } + common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) return scaleSet(appName, skipDeploy, false, processTuples) } diff --git a/plugins/ps/triggers.go b/plugins/ps/triggers.go index 9de576f28..fbd7ae922 100644 --- a/plugins/ps/triggers.go +++ b/plugins/ps/triggers.go @@ -91,6 +91,29 @@ func TriggerInstall() error { } } + for _, appName := range apps { + dokkuScaleFile := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE") + if common.FileExists(dokkuScaleFile) { + processTuples, err := common.FileToSlice(dokkuScaleFile) + if err != nil { + return err + } + + if err := scaleSet(appName, true, false, processTuples); err != nil { + return err + } + + os.Remove(dokkuScaleFile) + } + + dokkuScaleExtracted := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted") + if common.FileExists(dokkuScaleExtracted) { + if err := common.PropertyWrite("ps", appName, "can-scale", strconv.FormatBool(false)); err != nil { + return err + } + } + } + return nil } @@ -151,7 +174,8 @@ func TriggerPostCreate(appName string) error { return err } - return updateScalefile(appName, false, make(map[string]int)) + formations := FormationSlice{} + return updateScale(appName, false, formations) } // TriggerPostDelete destroys the ps properties for a given app container @@ -192,7 +216,7 @@ func TriggerPostStop(appName string) error { }) } -// TriggerPreDeploy ensures an app has an up to date scale file +// TriggerPreDeploy ensures an app has an up to date scale parameters func TriggerPreDeploy(appName string, imageTag string) error { image, err := common.GetDeployingAppImageName(appName, imageTag, "") if err != nil { @@ -207,7 +231,8 @@ func TriggerPreDeploy(appName string, imageTag string) error { return err } - if err := extractOrGenerateScalefile(appName, image); err != nil { + if err := updateScale(appName, false, FormationSlice{}); err != nil { + common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error())) return err } diff --git a/tests/unit/ps-general.bats b/tests/unit/ps-general.bats index cd237b151..7c8a5bc34 100644 --- a/tests/unit/ps-general.bats +++ b/tests/unit/ps-general.bats @@ -55,7 +55,7 @@ EOF assert_output "node worker.js" } -@test "(ps:scale) update DOKKU_SCALE from Procfile" { +@test "(ps:scale) update formations from Procfile" { local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM