fix: serialize concurrent property updates

Concurrent commands for the same app could each read a setting, change it, and write it back, so all of them exited successfully while silently dropping changes made by the others. Updates to app settings are now serialized so every change is kept, and readers never see a partially written value.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-10-21 21:21:57 -04:00
1 parent ba1d49576b
commit 9df45f6a6e
42 files changed
+1698 -642

No files matched your search

@@ -0,0 +1,72 @@
package dockeroptions
import (
"errors"
"fmt"
"sync"
"testing"
)
func TestAddDockerOptionToPhasesConcurrent(t *testing.T) {
const appName = "race-test"
setupReportEnv(t, appName)
const count = 20
var wg sync.WaitGroup
errs := make([]error, count)
for i := 0; i < count; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
errs[i] = AddDockerOptionToPhases(appName, []string{"deploy", "run"}, fmt.Sprintf("--label=race%d", i))
}(i)
}
wg.Wait()
if err := errors.Join(errs...); err != nil {
t.Fatalf("AddDockerOptionToPhases: %v", err)
}
for _, phase := range []string{"deploy", "run"} {
options, err := GetDockerOptionsForPhase(appName, phase)
if err != nil {
t.Fatalf("GetDockerOptionsForPhase(%s): %v", phase, err)
}
if len(options) != count {
t.Fatalf("%s phase has %d options, want %d: %v", phase, len(options), count, options)
}
}
}
func TestRemoveDockerOptionFromPhasesConcurrent(t *testing.T) {
const appName = "race-test"
setupReportEnv(t, appName)
const count = 20
for i := 0; i < count; i++ {
if err := AddDockerOptionToPhases(appName, []string{"deploy"}, fmt.Sprintf("--label=race%d", i)); err != nil {
t.Fatalf("AddDockerOptionToPhases: %v", err)
}
}
var wg sync.WaitGroup
errs := make([]error, count)
for i := 0; i < count; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
errs[i] = RemoveDockerOptionFromPhases(appName, []string{"deploy"}, fmt.Sprintf("--label=race%d", i))
}(i)
}
wg.Wait()
if err := errors.Join(errs...); err != nil {
t.Fatalf("RemoveDockerOptionFromPhases: %v", err)
}
options, err := GetDockerOptionsForPhase(appName, "deploy")
if err != nil {
t.Fatalf("GetDockerOptionsForPhase: %v", err)
}
if len(options) != 0 {
t.Fatalf("deploy phase still has options: %v", options)
}
}
+46 -66
View File
@@ -198,32 +198,17 @@ func SetDockerOptionForPhases(appName string, phases []string, name string, valu
// SetDockerOptionForProcessPhases sets a `--name=value` option for the specified
// process types and phases, replacing any existing entry with the same name.
func SetDockerOptionForProcessPhases(appName string, processTypes []string, phases []string, name string, value string) error {
if len(processTypes) == 0 {
processTypes = []string{DefaultProcessType}
}
for _, processType := range processTypes {
for _, phase := range phases {
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
if err != nil {
return err
}
newOptions := []string{}
for _, option := range options {
if strings.HasPrefix(option, fmt.Sprintf("--%s=", name)) {
continue
}
newOptions = append(newOptions, option)
}
newOptions = append(newOptions, fmt.Sprintf("--%s=%s", name, value))
sort.Strings(newOptions)
if err := writeDockerOptionsForProcessPhase(appName, processType, phase, newOptions); err != nil {
return err
return updateDockerOptionsForProcessPhases(appName, processTypes, phases, func(options []string) []string {
newOptions := []string{}
for _, option := range options {
if strings.HasPrefix(option, fmt.Sprintf("--%s=", name)) {
continue
}
newOptions = append(newOptions, option)
}
}
return nil
return append(newOptions, fmt.Sprintf("--%s=%s", name, value))
})
}
// AddDockerOptionToPhases adds an option to the default scope for the specified phases.
@@ -233,24 +218,9 @@ func AddDockerOptionToPhases(appName string, phases []string, option string) err
// AddDockerOptionToProcessPhases adds an option to the specified process types and phases.
func AddDockerOptionToProcessPhases(appName string, processTypes []string, phases []string, option string) error {
if len(processTypes) == 0 {
processTypes = []string{DefaultProcessType}
}
for _, processType := range processTypes {
for _, phase := range phases {
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
if err != nil {
return err
}
options = append(options, option)
sort.Strings(options)
if err := writeDockerOptionsForProcessPhase(appName, processType, phase, options); err != nil {
return err
}
}
}
return nil
return updateDockerOptionsForProcessPhases(appName, processTypes, phases, func(options []string) []string {
return append(options, option)
})
}
// GetDockerOptionsForPhase returns the docker options stored under the default
@@ -290,30 +260,15 @@ func RemoveDockerOptionFromPhases(appName string, phases []string, option string
// non-canonical form still matches the canonically re-serialized string the
// CLI hands down.
func RemoveDockerOptionFromProcessPhases(appName string, processTypes []string, phases []string, option string) error {
if len(processTypes) == 0 {
processTypes = []string{DefaultProcessType}
}
for _, processType := range processTypes {
for _, phase := range phases {
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
if err != nil {
return err
}
newOptions := []string{}
for _, opt := range options {
if !optionsEqual(opt, option) {
newOptions = append(newOptions, opt)
}
}
sort.Strings(newOptions)
if err := writeDockerOptionsForProcessPhase(appName, processType, phase, newOptions); err != nil {
return err
return updateDockerOptionsForProcessPhases(appName, processTypes, phases, func(options []string) []string {
newOptions := []string{}
for _, opt := range options {
if !optionsEqual(opt, option) {
newOptions = append(newOptions, opt)
}
}
}
return nil
return newOptions
})
}
// GetSpecifiedDockerOptionsForPhase returns the docker options for the specified
@@ -410,6 +365,31 @@ func splitPropertyKey(key string) (processType, phase string, ok bool) {
return processType, phase, true
}
func writeDockerOptionsForProcessPhase(appName, processType, phase string, options []string) error {
return common.PropertyListWrite("docker-options", appName, propertyKey(processType, phase), options)
// updateDockerOptionsForProcessPhases rewrites the stored options of every
// process type and phase combination with the sorted result of update, holding
// the app's docker-options property lock across every read and write so
// concurrent updates to the same app cannot drop each other's options. An
// empty processTypes list is treated as the default scope.
func updateDockerOptionsForProcessPhases(appName string, processTypes []string, phases []string, update func(options []string) []string) error {
if len(processTypes) == 0 {
processTypes = []string{DefaultProcessType}
}
return common.PropertyLock("docker-options", appName, func(p *common.LockedProperties) error {
for _, processType := range processTypes {
for _, phase := range phases {
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
if err != nil {
return err
}
options = update(options)
sort.Strings(options)
if err := p.ListWrite(propertyKey(processType, phase), options); err != nil {
return err
}
}
}
return nil
})
}
+52 -40
View File
@@ -390,36 +390,42 @@ func migrateTraefikLabelBackticks() error {
}
for _, appName := range apps {
properties, err := common.PropertyGetAll("docker-options", appName)
if err != nil {
return err
}
for key := range properties {
processType, phase, ok := splitPropertyKey(key)
if !ok {
continue
}
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
err := common.PropertyLock("docker-options", appName, func(p *common.LockedProperties) error {
properties, err := common.PropertyGetAll("docker-options", appName)
if err != nil {
return err
}
changed := false
for i, option := range options {
if fixed, ok := repairTraefikLabelBackticks(option); ok {
options[i] = fixed
changed = true
for key := range properties {
processType, phase, ok := splitPropertyKey(key)
if !ok {
continue
}
}
if changed {
if err := writeDockerOptionsForProcessPhase(appName, processType, phase, options); err != nil {
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
if err != nil {
return err
}
common.LogInfo1(fmt.Sprintf("Repaired docker-options label backticks for %s (%s %s)", appName, processType, phase))
changed := false
for i, option := range options {
if fixed, ok := repairTraefikLabelBackticks(option); ok {
options[i] = fixed
changed = true
}
}
if changed {
if err := p.ListWrite(key, options); err != nil {
return err
}
common.LogInfo1(fmt.Sprintf("Repaired docker-options label backticks for %s (%s %s)", appName, processType, phase))
}
}
return nil
})
if err != nil {
return err
}
}
@@ -494,31 +500,37 @@ func migrateNonCanonicalOptions() error {
}
for _, appName := range apps {
properties, err := common.PropertyGetAll("docker-options", appName)
if err != nil {
return err
}
for key := range properties {
processType, phase, ok := splitPropertyKey(key)
if !ok {
continue
}
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
err := common.PropertyLock("docker-options", appName, func(p *common.LockedProperties) error {
properties, err := common.PropertyGetAll("docker-options", appName)
if err != nil {
return err
}
canonical := canonicalizeOptionLines(options)
if slices.Equal(canonical, options) {
continue
}
for key := range properties {
processType, phase, ok := splitPropertyKey(key)
if !ok {
continue
}
if err := writeDockerOptionsForProcessPhase(appName, processType, phase, canonical); err != nil {
return err
options, err := GetDockerOptionsForProcessPhase(appName, processType, phase)
if err != nil {
return err
}
canonical := canonicalizeOptionLines(options)
if slices.Equal(canonical, options) {
continue
}
if err := p.ListWrite(key, canonical); err != nil {
return err
}
common.LogInfo1(fmt.Sprintf("Rewrote docker-options into canonical form for %s (%s %s)", appName, processType, phase))
}
common.LogInfo1(fmt.Sprintf("Rewrote docker-options into canonical form for %s (%s %s)", appName, processType, phase))
return nil
})
if err != nil {
return err
}
}