fix: serialize concurrent property updates
Concurrent commands for the same app could each read a setting, change it, and write it back, so all of them exited successfully while silently dropping changes made by the others. Updates to app settings are now serialized so every change is kept, and readers never see a partially written value.
This commit is contained in:
42 files changed
+1698
-642
No files matched your search
@@ -0,0 +1,62 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestConfigSetManyConcurrent(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
setupTestApp(t)
|
||||
|
||||
const count = 20
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, count)
|
||||
for i := 0; i < count; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
errs[i] = SetMany(testAppName, map[string]string{fmt.Sprintf("KEY%d", i): "value"}, false, false)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
Expect(errors.Join(errs...)).NotTo(HaveOccurred())
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
expectValue(testAppName, fmt.Sprintf("KEY%d", i), "value")
|
||||
}
|
||||
expectValue(testAppName, "testKey", "TESTING")
|
||||
}
|
||||
|
||||
func TestConfigUnsetManyConcurrent(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
setupTestApp(t)
|
||||
|
||||
const count = 20
|
||||
entries := map[string]string{}
|
||||
for i := 0; i < count; i++ {
|
||||
entries[fmt.Sprintf("KEY%d", i)] = "value"
|
||||
}
|
||||
Expect(SetMany(testAppName, entries, false, false)).To(Succeed())
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, count)
|
||||
for i := 0; i < count; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
errs[i] = UnsetMany(testAppName, []string{fmt.Sprintf("KEY%d", i)}, false)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
Expect(errors.Join(errs...)).NotTo(HaveOccurred())
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
expectNoValue(testAppName, fmt.Sprintf("KEY%d", i))
|
||||
}
|
||||
expectValue(testAppName, "testKey", "TESTING")
|
||||
}
|
||||
+85
-41
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
"github.com/dokku/dokku/plugins/common"
|
||||
@@ -32,10 +33,6 @@ func GetWithDefault(appName string, key string, defaultValue string) (value stri
|
||||
// SetMany variables in the environment. If appName is empty the global config is used. If restart is true the app is restarted.
|
||||
func SetMany(appName string, entries map[string]string, replace bool, restart bool) (err error) {
|
||||
global := appName == "" || appName == "--global"
|
||||
env, err := loadAppOrGlobalEnv(appName)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(entries))
|
||||
for k := range entries {
|
||||
if err = validateKey(k); err != nil {
|
||||
@@ -43,21 +40,28 @@ func SetMany(appName string, entries map[string]string, replace bool, restart bo
|
||||
}
|
||||
}
|
||||
|
||||
if replace {
|
||||
env.Clear()
|
||||
}
|
||||
for k, v := range entries {
|
||||
env.Set(k, v)
|
||||
keys = append(keys, k)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
changed, err := updateAppOrGlobalEnv(appName, func(env *Env) bool {
|
||||
if replace {
|
||||
env.Clear()
|
||||
}
|
||||
for k, v := range entries {
|
||||
env.Set(k, v)
|
||||
keys = append(keys, k)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
common.LogInfo1Quiet("Setting config vars")
|
||||
if os.Getenv("DOKKU_QUIET_OUTPUT") == "" {
|
||||
fmt.Println(prettyPrintEnvEntries(" ", entries))
|
||||
}
|
||||
if err := env.Write(); err != nil {
|
||||
return fmt.Errorf("Unable to write config vars: %s", err.Error())
|
||||
}
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed {
|
||||
triggerUpdate(appName, "set", keys)
|
||||
}
|
||||
if !global && restart && shouldRestart(appName) {
|
||||
@@ -69,29 +73,28 @@ func SetMany(appName string, entries map[string]string, replace bool, restart bo
|
||||
// UnsetMany a value in a config. If appName is empty the global config is used. If restart is true the app is restarted.
|
||||
func UnsetMany(appName string, keys []string, restart bool) (err error) {
|
||||
global := appName == "" || appName == "--global"
|
||||
env, err := loadAppOrGlobalEnv(appName)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var changed = false
|
||||
for _, k := range keys {
|
||||
if err = validateKey(k); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, k := range keys {
|
||||
if _, hasKey := env.Map()[k]; hasKey {
|
||||
common.LogInfo1Quiet(fmt.Sprintf("Unsetting %s", k))
|
||||
env.Unset(k)
|
||||
changed = true
|
||||
} else {
|
||||
common.LogInfo1Quiet(fmt.Sprintf("Skipping %s, it is not set in the environment", k))
|
||||
changed, err := updateAppOrGlobalEnv(appName, func(env *Env) bool {
|
||||
changed := false
|
||||
for _, k := range keys {
|
||||
if _, hasKey := env.Map()[k]; hasKey {
|
||||
common.LogInfo1Quiet(fmt.Sprintf("Unsetting %s", k))
|
||||
env.Unset(k)
|
||||
changed = true
|
||||
} else {
|
||||
common.LogInfo1Quiet(fmt.Sprintf("Skipping %s, it is not set in the environment", k))
|
||||
}
|
||||
}
|
||||
return changed
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed {
|
||||
if err := env.Write(); err != nil {
|
||||
return fmt.Errorf("Unable to write config vars: %s", err.Error())
|
||||
}
|
||||
triggerUpdate(appName, "unset", keys)
|
||||
}
|
||||
if !global && restart && shouldRestart(appName) {
|
||||
@@ -103,20 +106,19 @@ func UnsetMany(appName string, keys []string, restart bool) (err error) {
|
||||
// UnsetAll removes all config keys
|
||||
func UnsetAll(appName string, restart bool) (err error) {
|
||||
global := appName == "" || appName == "--global"
|
||||
env, err := loadAppOrGlobalEnv(appName)
|
||||
changed, err := updateAppOrGlobalEnv(appName, func(env *Env) bool {
|
||||
changed := false
|
||||
for k := range env.Map() {
|
||||
common.LogInfo1Quiet(fmt.Sprintf("Unsetting %s", k))
|
||||
env.Unset(k)
|
||||
changed = true
|
||||
}
|
||||
return changed
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var changed = false
|
||||
for k := range env.Map() {
|
||||
common.LogInfo1Quiet(fmt.Sprintf("Unsetting %s", k))
|
||||
env.Unset(k)
|
||||
changed = true
|
||||
return err
|
||||
}
|
||||
if changed {
|
||||
if err := env.Write(); err != nil {
|
||||
return fmt.Errorf("Unable to write config vars: %s", err.Error())
|
||||
}
|
||||
triggerUpdate(appName, "clear", []string{})
|
||||
}
|
||||
if !global && restart && shouldRestart(appName) {
|
||||
@@ -125,6 +127,48 @@ func UnsetAll(appName string, restart bool) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// updateAppOrGlobalEnv loads the app or global environment, passes it to
|
||||
// update and writes it back when update reports a change, returning whether it
|
||||
// did. The directory holding the environment file stays locked from the load
|
||||
// to the write, so concurrent config changes cannot drop each other's keys.
|
||||
// Triggers reacting to the change must run after this returns, as a trigger
|
||||
// that changes the same environment would otherwise wait on the lock forever.
|
||||
func updateAppOrGlobalEnv(appName string, update func(env *Env) bool) (bool, error) {
|
||||
filename := getGlobalFile()
|
||||
if appName != "" && appName != "--global" {
|
||||
var err error
|
||||
if filename, err = getAppFile(appName); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
changed := false
|
||||
apply := func() error {
|
||||
env, err := loadAppOrGlobalEnv(appName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !update(env) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := env.Write(); err != nil {
|
||||
return fmt.Errorf("Unable to write config vars: %s", err.Error())
|
||||
}
|
||||
changed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
dir := filepath.Dir(filename)
|
||||
if !common.DirectoryExists(dir) {
|
||||
return changed, apply()
|
||||
}
|
||||
|
||||
err := common.LockDirectory(dir, apply)
|
||||
return changed, err
|
||||
}
|
||||
|
||||
func shouldRestart(appName string) bool {
|
||||
results, _ := common.CallPlugnTrigger(common.PlugnTriggerInput{
|
||||
Trigger: "ps-get-property",
|
||||
|
||||
@@ -54,12 +54,14 @@ func migrateGlobalEnv() error {
|
||||
}
|
||||
|
||||
oldGlobalEnvFile := filepath.Join(common.MustGetEnv("DOKKU_ROOT"), "ENV")
|
||||
globalEnv, err := LoadGlobalEnv()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to load global environment: %s", err.Error())
|
||||
}
|
||||
return common.LockDirectory(filepath.Dir(getGlobalFile()), func() error {
|
||||
globalEnv, err := LoadGlobalEnv()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to load global environment: %s", err.Error())
|
||||
}
|
||||
|
||||
return drainLegacyEnvFile("--global", oldGlobalEnvFile, globalEnv)
|
||||
return drainLegacyEnvFile("--global", oldGlobalEnvFile, globalEnv)
|
||||
})
|
||||
}
|
||||
|
||||
// migrateAppEnv drains $DOKKU_ROOT/<app>/ENV into the app's config property path.
|
||||
@@ -72,13 +74,20 @@ func migrateAppEnv(appName string) error {
|
||||
return fmt.Errorf("Unable to setup app config directory: %s", err.Error())
|
||||
}
|
||||
|
||||
env, err := LoadAppEnv(appName)
|
||||
appFile, err := getAppFile(appName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to load app environment: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
oldEnvFile := filepath.Join(common.AppRoot(appName), "ENV")
|
||||
return drainLegacyEnvFile(appName, oldEnvFile, env)
|
||||
return common.LockDirectory(filepath.Dir(appFile), func() error {
|
||||
env, err := LoadAppEnv(appName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to load app environment: %s", err.Error())
|
||||
}
|
||||
|
||||
return drainLegacyEnvFile(appName, oldEnvFile, env)
|
||||
})
|
||||
}
|
||||
|
||||
// drainLegacyEnvFile merges the legacy ENV file at oldEnvFile into env, records
|
||||
|
||||
Reference in new issue
Block a user