refactor: write DOKKU_SCALE contents to scale property

- remove extraction of DOKKU_SCALE
- migrate to property system
- rewrite internals to use a FormationSlice instead of a map
This commit is contained in:
Jose Diaz-Gonzalez committed 2021-08-01 17:47:04 -04:00
1 parent 8fc0cb67ec
commit 401683a305
6 files changed
+105 -192

No files matched your search

+11 -18
View File
@@ -173,6 +173,15 @@ func PropertyListAdd(pluginName string, appName string, property string, value s
lines = append(lines, value) lines = append(lines, value)
} }
return PropertyListWrite(pluginName, appName, property, lines)
}
// PropertyListWrite completely rewrites a list property
func PropertyListWrite(pluginName string, appName string, property string, values []string) error {
if err := propertyTouch(pluginName, appName, property); err != nil {
return err
}
propertyPath := getPropertyPath(pluginName, appName, property) propertyPath := getPropertyPath(pluginName, appName, property)
file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600) file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600)
if err != nil { if err != nil {
@@ -180,7 +189,7 @@ func PropertyListAdd(pluginName string, appName string, property string, value s
} }
w := bufio.NewWriter(file) w := bufio.NewWriter(file)
for _, line := range lines { for _, line := range values {
fmt.Fprintln(w, line) fmt.Fprintln(w, line)
} }
if err = w.Flush(); err != nil { if err = w.Flush(); err != nil {
@@ -389,23 +398,7 @@ func PropertyListSet(pluginName string, appName string, property string, value s
} }
} }
propertyPath := getPropertyPath(pluginName, appName, property) return PropertyListWrite(pluginName, appName, property, lines)
file, err := os.OpenFile(propertyPath, os.O_RDWR|os.O_TRUNC, 0600)
if err != nil {
return err
}
w := bufio.NewWriter(file)
for _, line := range lines {
fmt.Fprintln(w, line)
}
if err = w.Flush(); err != nil {
return fmt.Errorf("Unable to write %s config value %s.%s: %s", pluginName, appName, property, err.Error())
}
file.Chmod(0600)
SetPermissions(propertyPath, 0600)
return nil
} }
// propertyTouch ensures a given application property file exists // propertyTouch ensures a given application property file exists
+10
View File
@@ -159,6 +159,16 @@ func main() {
fmt.Fprintln(os.Stderr, err.Error()) fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1) os.Exit(1)
} }
case "lwrite":
_, args := common.ShiftString(flag.Args())
appName, args := common.ShiftString(args)
property, args := common.ShiftString(args)
_, values := common.ShiftString(args)
err := common.PropertyListWrite(pluginName, appName, property, values)
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
case "rpush": case "rpush":
appName := flag.Arg(2) appName := flag.Arg(2)
property := flag.Arg(3) property := flag.Arg(3)
+52 -159
View File
@@ -47,64 +47,6 @@ func extractProcfile(appName, image string) error {
return nil return nil
} }
func extractOrGenerateScalefile(appName string, image string) error {
destination := getScalefilePath(appName)
extracted := getScalefileExtractedPath(appName)
previouslyExtracted := common.FileExists(extracted)
if err := common.CopyFromImage(appName, image, "DOKKU_SCALE", extracted); err != nil {
if previouslyExtracted {
os.Remove(destination)
}
os.Remove(extracted)
} else if err := common.CopyFile(extracted, destination); err != nil {
return err
}
if common.FileExists(destination) {
common.LogInfo1Quiet("DOKKU_SCALE file exists")
return updateScalefile(appName, false, make(map[string]int))
}
common.LogInfo1Quiet("DOKKU_SCALE file not found in app image. Generating one based on Procfile...")
if err := updateScalefile(appName, false, make(map[string]int)); err != nil {
common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error()))
return err
}
return nil
}
func generateScalefile(appName string) error {
destination := getScalefilePath(appName)
procfilePath := getProcfilePath(appName)
content := []string{"web=1"}
if !common.FileExists(procfilePath) {
return common.WriteSliceToFile(destination, content)
}
lines, err := common.FileToSlice(procfilePath)
if err != nil {
return common.WriteSliceToFile(destination, content)
}
content = []string{}
for _, line := range lines {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.Split(line, ":")
count := 0
if parts[0] == "web" {
count = 1
}
content = append(content, fmt.Sprintf("%s=%d", parts[0], count))
}
return common.WriteSliceToFile(destination, content)
}
func getProcessStatus(appName string) map[string]string { func getProcessStatus(appName string) map[string]string {
statuses := make(map[string]string) statuses := make(map[string]string)
containerFiles := common.ListFilesWithPrefix(common.AppRoot(appName), "CONTAINER.") containerFiles := common.ListFilesWithPrefix(common.AppRoot(appName), "CONTAINER.")
@@ -183,19 +125,6 @@ func getRunningState(appName string) string {
return strings.Split(strings.TrimSpace(string(b[:])), " ")[1] return strings.Split(strings.TrimSpace(string(b[:])), " ")[1]
} }
func getScalefilePath(appName string) string {
return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE")
}
func getScalefileExtractedPath(appName string) string {
return filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted")
}
func hasScaleFile(appName string) bool {
scalefilePath := getScalefilePath(appName)
return common.FileExists(scalefilePath)
}
func hasProcfile(appName string) bool { func hasProcfile(appName string) bool {
procfilePath := getProcfilePath(appName) procfilePath := getProcfilePath(appName)
return common.FileExists(procfilePath) return common.FileExists(procfilePath)
@@ -220,25 +149,28 @@ func isValidRestartPolicy(policy string) bool {
return strings.HasPrefix(policy, "on-failure:") return strings.HasPrefix(policy, "on-failure:")
} }
func parseProcessTuples(processTuples []string) (map[string]int, error) { func parseProcessTuples(processTuples []string) (FormationSlice, error) {
scale := make(map[string]int) formations := FormationSlice{}
for _, processTuple := range processTuples { for _, processTuple := range processTuples {
s := strings.Split(processTuple, "=") s := strings.Split(processTuple, "=")
if len(s) == 1 { if len(s) == 1 {
return scale, fmt.Errorf("Missing count for process type %s", processTuple) return formations, fmt.Errorf("Missing count for process type %s", processTuple)
} }
processType := s[0] processType := s[0]
count, err := strconv.Atoi(s[1]) quantity, err := strconv.Atoi(s[1])
if err != nil { if err != nil {
return scale, fmt.Errorf("Invalid count for process type %s", s[0]) return formations, fmt.Errorf("Invalid count for process type %s", s[0])
} }
scale[processType] = count formations = append(formations, &Formation{
ProcessType: processType,
Quantity: quantity,
})
} }
return scale, nil return formations, nil
} }
func processesInProcfile(procfilePath string) (map[string]bool, error) { func processesInProcfile(procfilePath string) (map[string]bool, error) {
@@ -266,63 +198,14 @@ func processesInProcfile(procfilePath string) (map[string]bool, error) {
return processes, nil return processes, nil
} }
func readScaleFile(appName string) (map[string]int, error) {
scale := make(map[string]int)
if !hasScaleFile(appName) {
return scale, errors.New("No DOKKU_SCALE file found")
}
scalefilePath := getScalefilePath(appName)
lines, err := common.FileToSlice(scalefilePath)
if err != nil {
return scale, err
}
for i, line := range lines {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
s := strings.Split(line, "=")
if len(s) != 2 {
common.LogWarn(fmt.Sprintf("Invalid scale entry on line %d, skipping", i))
continue
}
processType := s[0]
count, err := strconv.Atoi(s[1])
if err != nil {
common.LogWarn(fmt.Sprintf("Invalid count on line %d, skipping", i))
continue
}
scale[processType] = count
}
if len(scale) == 0 {
common.LogWarn("No valid entries found in scale file, defaulting to web=1")
scale["web"] = 1
}
return scale, nil
}
func getFormations(appName string) (FormationSlice, error) { func getFormations(appName string) (FormationSlice, error) {
scale := FormationSlice{} formations := FormationSlice{}
data, err := readScaleFile(appName) processTuples, err := common.PropertyListGet("ps", appName, "scale")
if err != nil { if err != nil {
return scale, err return formations, err
} }
for processType, quantity := range data { return parseProcessTuples(processTuples)
f := Formation{
ProcessType: processType,
Quantity: quantity,
}
scale = append(scale, &f)
}
return scale, nil
} }
func removeProcfile(appName string) error { func removeProcfile(appName string) error {
@@ -382,13 +265,12 @@ func scaleReport(appName string) error {
} }
func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error { func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples []string) error {
scale, err := parseProcessTuples(processTuples) formations, err := parseProcessTuples(processTuples)
if err != nil { if err != nil {
return err return err
} }
common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " "))) if err := updateScale(appName, clearExisting, formations); err != nil {
if err := updateScalefile(appName, clearExisting, scale); err != nil {
return err return err
} }
@@ -408,17 +290,15 @@ func scaleSet(appName string, skipDeploy bool, clearExisting bool, processTuples
return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...) return common.PlugnTrigger("release-and-deploy", []string{appName, imageTag}...)
} }
func updateScalefile(appName string, clearExisting bool, scaleUpdates map[string]int) error { func updateScale(appName string, clearExisting bool, formationUpdates FormationSlice) error {
if !hasScaleFile(appName) { formations := FormationSlice{}
if err := generateScalefile(appName); err != nil { if !clearExisting {
processTuples, err := common.PropertyListGet("ps", appName, "scale")
if err != nil {
return err return err
} }
}
scale := make(map[string]int) formations, err = parseProcessTuples(processTuples)
if !clearExisting {
var err error
scale, err = readScaleFile(appName)
if err != nil { if err != nil {
return err return err
} }
@@ -435,30 +315,43 @@ func updateScalefile(appName string, clearExisting bool, scaleUpdates map[string
} }
} }
for processType, count := range scaleUpdates { foundProcessTypes := map[string]bool{}
if procfileExists && !validProcessTypes[processType] && count != 0 { updatedFormation := FormationSlice{}
return fmt.Errorf("%s is not a valid process name to scale up", processType) for _, formation := range formationUpdates {
if procfileExists && !validProcessTypes[formation.ProcessType] && formation.Quantity != 0 {
return fmt.Errorf("%s is not a valid process name to scale up", formation.ProcessType)
} }
scale[processType] = count
foundProcessTypes[formation.ProcessType] = true
updatedFormation = append(updatedFormation, &Formation{
ProcessType: formation.ProcessType,
Quantity: formation.Quantity,
})
}
for _, formation := range formations {
foundProcessTypes[formation.ProcessType] = true
updatedFormation = append(updatedFormation, &Formation{
ProcessType: formation.ProcessType,
Quantity: formation.Quantity,
})
} }
for processType := range validProcessTypes { for processType := range validProcessTypes {
count, ok := scale[processType] if foundProcessTypes[processType] {
if !ok { continue
count = 0
} }
scale[processType] = count
updatedFormation = append(updatedFormation, &Formation{
ProcessType: processType,
Quantity: 0,
})
} }
content := []string{} values := []string{}
for processType, count := range scale { for _, formation := range formations {
content = append(content, fmt.Sprintf("%s=%d", processType, count)) values = append(values, fmt.Sprintf("%s=%d", formation.ProcessType, formation.Quantity))
} }
scalefilePath := getScalefilePath(appName) return common.PropertyListWrite("ps", appName, "scale", values)
if err := common.WriteSliceToFile(scalefilePath, content); err != nil {
return err
}
return nil
} }
+3 -11
View File
@@ -4,6 +4,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"path/filepath" "path/filepath"
"strings"
dockeroptions "github.com/dokku/dokku/plugins/docker-options" dockeroptions "github.com/dokku/dokku/plugins/docker-options"
@@ -120,24 +121,15 @@ func CommandScale(appName string, skipDeploy bool, processTuples []string) error
return err return err
} }
procfilePath := getProcfilePath(appName)
if !hasScaleFile(appName) || common.FileExists(procfilePath) {
update := func() error {
return updateScalefile(appName, false, make(map[string]int))
}
if err := common.SuppressOutput(update); err != nil {
return err
}
}
if len(processTuples) == 0 { if len(processTuples) == 0 {
return scaleReport(appName) return scaleReport(appName)
} }
if !canScaleApp(appName) { if !canScaleApp(appName) {
return fmt.Errorf("App %s contains DOKKU_SCALE file and cannot be manually scaled", appName) return fmt.Errorf("App %s contains an app.json file with a formations key and cannot be manually scaled", appName)
} }
common.LogInfo1(fmt.Sprintf("Scaling %s processes: %s", appName, strings.Join(processTuples, " ")))
return scaleSet(appName, skipDeploy, false, processTuples) return scaleSet(appName, skipDeploy, false, processTuples)
} }
+28 -3
View File
@@ -91,6 +91,29 @@ func TriggerInstall() error {
} }
} }
for _, appName := range apps {
dokkuScaleFile := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE")
if common.FileExists(dokkuScaleFile) {
processTuples, err := common.FileToSlice(dokkuScaleFile)
if err != nil {
return err
}
if err := scaleSet(appName, true, false, processTuples); err != nil {
return err
}
os.Remove(dokkuScaleFile)
}
dokkuScaleExtracted := filepath.Join(common.AppRoot(appName), "DOKKU_SCALE.extracted")
if common.FileExists(dokkuScaleExtracted) {
if err := common.PropertyWrite("ps", appName, "can-scale", strconv.FormatBool(false)); err != nil {
return err
}
}
}
return nil return nil
} }
@@ -151,7 +174,8 @@ func TriggerPostCreate(appName string) error {
return err return err
} }
return updateScalefile(appName, false, make(map[string]int)) formations := FormationSlice{}
return updateScale(appName, false, formations)
} }
// TriggerPostDelete destroys the ps properties for a given app container // TriggerPostDelete destroys the ps properties for a given app container
@@ -192,7 +216,7 @@ func TriggerPostStop(appName string) error {
}) })
} }
// TriggerPreDeploy ensures an app has an up to date scale file // TriggerPreDeploy ensures an app has an up to date scale parameters
func TriggerPreDeploy(appName string, imageTag string) error { func TriggerPreDeploy(appName string, imageTag string) error {
image, err := common.GetDeployingAppImageName(appName, imageTag, "") image, err := common.GetDeployingAppImageName(appName, imageTag, "")
if err != nil { if err != nil {
@@ -207,7 +231,8 @@ func TriggerPreDeploy(appName string, imageTag string) error {
return err return err
} }
if err := extractOrGenerateScalefile(appName, image); err != nil { if err := updateScale(appName, false, FormationSlice{}); err != nil {
common.LogDebug(fmt.Sprintf("Error generating scale file: %s", err.Error()))
return err return err
} }
+1 -1
View File
@@ -55,7 +55,7 @@ EOF
assert_output "node worker.js" assert_output "node worker.js"
} }
@test "(ps:scale) update DOKKU_SCALE from Procfile" { @test "(ps:scale) update formations from Procfile" {
local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX")
trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM