fix: serialize concurrent property updates

Concurrent commands for the same app could each read a setting, change it, and write it back, so all of them exited successfully while silently dropping changes made by the others. Updates to app settings are now serialized so every change is kept, and readers never see a partially written value.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-10-21 21:21:57 -04:00
1 parent ba1d49576b
commit 9df45f6a6e
42 files changed
+1698 -642

No files matched your search

+77 -53
View File
@@ -105,6 +105,11 @@ func LoadAttachments(appName string) ([]*Attachment, error) {
return nil, fmt.Errorf("unable to read storage attachments for %q: %w", appName, err)
}
return decodeAttachments(appName, lines)
}
// decodeAttachments parses the stored attachment lines of an app.
func decodeAttachments(appName string, lines []string) ([]*Attachment, error) {
attachments := []*Attachment{}
for _, line := range lines {
line = strings.TrimSpace(line)
@@ -122,15 +127,44 @@ func LoadAttachments(appName string) ([]*Attachment, error) {
// SaveAttachments overwrites the entire attachment list for an app.
func SaveAttachments(appName string, attachments []*Attachment) error {
lines, err := encodeAttachments(appName, attachments)
if err != nil {
return err
}
return common.PropertyListWrite(PluginName, appName, AttachmentsProperty, lines)
}
// UpdateAttachments loads the attachment list for an app, passes it to fn and
// saves the list fn returns, holding the app's storage property lock across
// the read and the write so concurrent mounts and unmounts cannot drop each
// other's changes. Nothing is saved when fn returns an error.
func UpdateAttachments(appName string, fn func(attachments []*Attachment) ([]*Attachment, error)) error {
return common.PropertyListUpdate(PluginName, appName, AttachmentsProperty, func(lines []string) ([]string, error) {
attachments, err := decodeAttachments(appName, lines)
if err != nil {
return nil, err
}
attachments, err = fn(attachments)
if err != nil {
return nil, err
}
return encodeAttachments(appName, attachments)
})
}
// encodeAttachments serializes attachments into stored attachment lines.
func encodeAttachments(appName string, attachments []*Attachment) ([]string, error) {
lines := make([]string, 0, len(attachments))
for _, attachment := range attachments {
data, err := json.Marshal(attachment)
if err != nil {
return fmt.Errorf("unable to encode storage attachment for %q: %w", appName, err)
return nil, fmt.Errorf("unable to encode storage attachment for %q: %w", appName, err)
}
lines = append(lines, string(data))
}
return common.PropertyListWrite(PluginName, appName, AttachmentsProperty, lines)
return lines, nil
}
// ensureContainerPathFree rejects a candidate whose container path is already
@@ -161,26 +195,22 @@ func AddAttachment(appName string, attachment *Attachment) error {
return err
}
attachments, err := LoadAttachments(appName)
if err != nil {
return err
}
for _, existing := range attachments {
if existing.EntryName == attachment.EntryName &&
existing.ContainerPath == attachment.ContainerPath &&
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
return fmt.Errorf("storage entry %q is already mounted at %q for process type %q on app %q",
attachment.EntryName, attachment.ContainerPath, attachment.EffectiveProcessType(), appName)
return UpdateAttachments(appName, func(attachments []*Attachment) ([]*Attachment, error) {
for _, existing := range attachments {
if existing.EntryName == attachment.EntryName &&
existing.ContainerPath == attachment.ContainerPath &&
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
return nil, fmt.Errorf("storage entry %q is already mounted at %q for process type %q on app %q",
attachment.EntryName, attachment.ContainerPath, attachment.EffectiveProcessType(), appName)
}
}
}
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
return err
}
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
return nil, err
}
attachments = append(attachments, attachment)
return SaveAttachments(appName, attachments)
return append(attachments, attachment), nil
})
}
// UpsertAttachment inserts an attachment, or updates the mount-time fields
@@ -201,49 +231,43 @@ func UpsertAttachment(appName string, attachment *Attachment) (bool, error) {
return false, err
}
attachments, err := LoadAttachments(appName)
created := false
err := UpdateAttachments(appName, func(attachments []*Attachment) ([]*Attachment, error) {
for _, existing := range attachments {
if existing.EntryName == attachment.EntryName &&
existing.ContainerPath == attachment.ContainerPath &&
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
existing.Phases = attachment.Phases
existing.Subpath = attachment.Subpath
existing.Readonly = attachment.Readonly
existing.VolumeOptions = attachment.VolumeOptions
existing.VolumeChown = attachment.VolumeChown
return attachments, nil
}
}
// Only the append branch needs the overlap check: rewriting an existing
// tuple in place would otherwise trip over the attachment it is updating.
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
return nil, err
}
created = true
return append(attachments, attachment), nil
})
if err != nil {
return false, err
}
for _, existing := range attachments {
if existing.EntryName == attachment.EntryName &&
existing.ContainerPath == attachment.ContainerPath &&
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
existing.Phases = attachment.Phases
existing.Subpath = attachment.Subpath
existing.Readonly = attachment.Readonly
existing.VolumeOptions = attachment.VolumeOptions
existing.VolumeChown = attachment.VolumeChown
return false, SaveAttachments(appName, attachments)
}
}
// Only the append branch needs the overlap check: rewriting an existing
// tuple in place would otherwise trip over the attachment it is updating.
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
return false, err
}
attachments = append(attachments, attachment)
return true, SaveAttachments(appName, attachments)
return created, nil
}
// RemoveAttachment removes the attachment matching the given entry and
// optional container path. If containerPath is empty and there is more
// than one match, returns an error.
func RemoveAttachment(appName string, entryName string, containerPath string) error {
attachments, err := LoadAttachments(appName)
if err != nil {
return err
}
keep, err := removeMatchingAttachments(attachments, appName, entryName, containerPath)
if err != nil {
return err
}
return SaveAttachments(appName, keep)
return UpdateAttachments(appName, func(attachments []*Attachment) ([]*Attachment, error) {
return removeMatchingAttachments(attachments, appName, entryName, containerPath)
})
}
// removeMatchingAttachments returns the given list with the attachments
+17 -22
View File
@@ -274,32 +274,27 @@ func CommandSet(input CommandSetInput) error {
if !EntryExists(input.Name) {
return fmt.Errorf("storage entry %q does not exist", input.Name)
}
entry, err := LoadEntry(input.Name)
if err != nil {
return err
}
touchesDirectory := false
for _, change := range input.Changes {
if err := applyPropertyChange(entry, change); err != nil {
return err
entry, err := UpdateEntry(input.Name, func(entry *Entry) error {
for _, change := range input.Changes {
if err := applyPropertyChange(entry, change); err != nil {
return err
}
if change.Property == "chown" || change.Property == "mode" {
touchesDirectory = true
}
}
if change.Property == "chown" || change.Property == "mode" {
touchesDirectory = true
if input.Annotations != nil {
entry.Annotations = input.Annotations
}
if input.Labels != nil {
entry.Labels = input.Labels
}
}
if input.Annotations != nil {
entry.Annotations = input.Annotations
}
if input.Labels != nil {
entry.Labels = input.Labels
}
if err := entry.Validate(); err != nil {
return err
}
if err := SaveEntry(entry); err != nil {
return entry.Validate()
})
if err != nil {
return err
}
// Only converge the directory when the caller actually asked to change
+19 -24
View File
@@ -76,32 +76,27 @@ func setEntryMapKey(field entryMapField, name string, key string, value string)
return fmt.Errorf("storage entry %q does not exist", name)
}
entry, err := LoadEntry(name)
if err != nil {
return err
}
values := field.Get(entry)
if value == "" {
delete(values, key)
common.LogInfo2Quiet(fmt.Sprintf("Unsetting %s %s", field.Singular, key))
} else {
if values == nil {
values = map[string]string{}
entry, err := UpdateEntry(name, func(entry *Entry) error {
values := field.Get(entry)
if value == "" {
delete(values, key)
common.LogInfo2Quiet(fmt.Sprintf("Unsetting %s %s", field.Singular, key))
} else {
if values == nil {
values = map[string]string{}
}
values[key] = value
common.LogInfo2Quiet(fmt.Sprintf("Setting %s %s to %s", field.Singular, key, value))
}
values[key] = value
common.LogInfo2Quiet(fmt.Sprintf("Setting %s %s to %s", field.Singular, key, value))
}
if len(values) == 0 {
// Drop the empty map so the omitempty tag keeps it out of the JSON.
values = nil
}
field.Set(entry, values)
if len(values) == 0 {
// Drop the empty map so the omitempty tag keeps it out of the JSON.
values = nil
}
field.Set(entry, values)
if err := entry.Validate(); err != nil {
return err
}
if err := SaveEntry(entry); err != nil {
return entry.Validate()
})
if err != nil {
return err
}
+92
View File
@@ -0,0 +1,92 @@
package storage
import (
"errors"
"fmt"
"sync"
"testing"
. "github.com/onsi/gomega"
)
func TestAddAttachmentConcurrent(t *testing.T) {
RegisterTestingT(t)
withTempLibRoot(t)
const count = 20
var wg sync.WaitGroup
errs := make([]error, count)
for i := 0; i < count; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
errs[i] = AddAttachment("demo", &Attachment{
EntryName: fmt.Sprintf("demo-data-%d", i),
ContainerPath: fmt.Sprintf("/data/%d", i),
Phases: []string{PhaseDeploy, PhaseRun},
ProcessType: DefaultProcessType,
})
}(i)
}
wg.Wait()
Expect(errors.Join(errs...)).NotTo(HaveOccurred())
attachments, err := LoadAttachments("demo")
Expect(err).NotTo(HaveOccurred())
Expect(attachments).To(HaveLen(count))
}
func TestUpdateEntryConcurrent(t *testing.T) {
RegisterTestingT(t)
withTempLibRoot(t)
Expect(SaveEntry(&Entry{
Name: "demo-data",
Scheduler: SchedulerK3s,
Size: "2Gi",
})).To(Succeed())
const count = 20
var wg sync.WaitGroup
errs := make([]error, count)
for i := 0; i < count; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
_, errs[i] = UpdateEntry("demo-data", func(entry *Entry) error {
if entry.Labels == nil {
entry.Labels = map[string]string{}
}
entry.Labels[fmt.Sprintf("label-%d", i)] = "value"
return nil
})
}(i)
}
wg.Wait()
Expect(errors.Join(errs...)).NotTo(HaveOccurred())
entry, err := LoadEntry("demo-data")
Expect(err).NotTo(HaveOccurred())
Expect(entry.Labels).To(HaveLen(count))
}
func TestUpdateEntryErrorLeavesEntryUnchanged(t *testing.T) {
RegisterTestingT(t)
withTempLibRoot(t)
Expect(SaveEntry(&Entry{
Name: "demo-data",
Scheduler: SchedulerK3s,
Size: "2Gi",
})).To(Succeed())
_, err := UpdateEntry("demo-data", func(entry *Entry) error {
entry.Size = "5Gi"
return errors.New("boom")
})
Expect(err).To(MatchError("boom"))
entry, err := LoadEntry("demo-data")
Expect(err).NotTo(HaveOccurred())
Expect(entry.Size).To(Equal("2Gi"))
}
+34 -1
View File
@@ -67,7 +67,7 @@ var (
)
// Entry is the source of truth for a storage volume. One file per entry
// lives at $DOKKU_LIB_ROOT/config/storage/entries/<name>.json.
// lives at $DOKKU_LIB_ROOT/data/storage-registry/entries/<name>.json.
type Entry struct {
Name string `json:"name"`
Scheduler string `json:"scheduler"`
@@ -162,6 +162,39 @@ func SaveEntry(entry *Entry) error {
return nil
}
// UpdateEntry loads a registered entry, passes it to fn and saves it, holding
// the entries directory lock across the read and the write so concurrent
// edits to one entry cannot drop each other's changes. Nothing is saved when
// fn returns an error. The saved entry is returned.
func UpdateEntry(name string, fn func(entry *Entry) error) (*Entry, error) {
if err := EnsureEntriesDirectory(); err != nil {
return nil, err
}
var updated *Entry
err := common.LockDirectory(EntriesDirectory(), func() error {
entry, err := LoadEntry(name)
if err != nil {
return err
}
if err := fn(entry); err != nil {
return err
}
if err := SaveEntry(entry); err != nil {
return err
}
updated = entry
return nil
})
if err != nil {
return nil, err
}
return updated, nil
}
// DeleteEntry removes the on-disk record for an entry.
func DeleteEntry(name string) error {
if err := os.Remove(entryPath(name)); err != nil && !os.IsNotExist(err) {
+7 -8
View File
@@ -217,19 +217,18 @@ func migrateMount(appName string, processType string, mount string, phases []str
VolumeOptions: parsed.VolumeOptions,
}
existing, err := LoadAttachments(appName)
if err != nil {
return err
}
// A named scope whose mount the default scope already covers needs no
// attachment of its own - the default one reaches that process anyway, and
// recording both would manufacture two -v flags for one container path.
// The docker-options line is still drained so it stops being emitted twice.
if !attachmentExists(existing, attachment) && !coveredByDefaultScope(existing, attachment) {
existing = append(existing, attachment)
if err := SaveAttachments(appName, existing); err != nil {
return err
err := UpdateAttachments(appName, func(existing []*Attachment) ([]*Attachment, error) {
if attachmentExists(existing, attachment) || coveredByDefaultScope(existing, attachment) {
return existing, nil
}
return append(existing, attachment), nil
})
if err != nil {
return err
}
mountedPhases := []string{}
+48 -58
View File
@@ -390,34 +390,30 @@ func replaceMounts(input CommandMountInput) error {
}
}
existing, err := LoadAttachments(input.AppName)
err := UpdateAttachments(input.AppName, func(existing []*Attachment) ([]*Attachment, error) {
retained := []*Attachment{}
for _, attachment := range existing {
if attachment.EffectiveProcessType() != processType {
retained = append(retained, attachment)
}
}
// The declared set is already unique on container path within its own
// scope; what is left to check is the scopes this call does not replace,
// so a declared path cannot collide with one held by another process type.
// Checking before the write keeps the all-or-nothing contract.
for _, attachment := range declared {
if err := ensureContainerPathFree(input.AppName, retained, attachment); err != nil {
return nil, err
}
}
return append(retained, declared...), nil
})
if err != nil {
return err
}
retained := []*Attachment{}
for _, attachment := range existing {
if attachment.EffectiveProcessType() != processType {
retained = append(retained, attachment)
}
}
// The declared set is already unique on container path within its own
// scope; what is left to check is the scopes this call does not replace,
// so a declared path cannot collide with one held by another process type.
// Checking before the write keeps the all-or-nothing contract.
for _, attachment := range declared {
if err := ensureContainerPathFree(input.AppName, retained, attachment); err != nil {
return err
}
}
attachments := append(retained, declared...)
if err := SaveAttachments(input.AppName, attachments); err != nil {
return err
}
common.LogInfo1(fmt.Sprintf("Storage mounts for process type %s replaced on %s", processType, input.AppName))
return nil
}
@@ -510,30 +506,27 @@ func resolveUnmountTarget(mount string, containerDir string) (unmountTarget, err
// written, so an argument naming a mount the app does not have leaves the rest
// of the set in place rather than removing a prefix of it.
func unmountMounts(input CommandUnmountInput) error {
attachments, err := LoadAttachments(input.AppName)
if err != nil {
return err
}
for _, mount := range input.Mounts {
target, err := resolveUnmountTarget(mount, input.ContainerDir)
if err != nil {
return err
}
attachments, err = removeMatchingAttachments(attachments, input.AppName, target.EntryName, target.ContainerPath)
if err != nil {
// The legacy form historically said "Mount path does not
// exist." - preserve that exact wording so existing automation
// and the bats suite keep matching.
if target.Legacy && strings.Contains(err.Error(), "is not mounted") {
return errors.New("Mount path does not exist.")
return UpdateAttachments(input.AppName, func(attachments []*Attachment) ([]*Attachment, error) {
for _, mount := range input.Mounts {
target, err := resolveUnmountTarget(mount, input.ContainerDir)
if err != nil {
return nil, err
}
return err
}
}
return SaveAttachments(input.AppName, attachments)
attachments, err = removeMatchingAttachments(attachments, input.AppName, target.EntryName, target.ContainerPath)
if err != nil {
// The legacy form historically said "Mount path does not
// exist." - preserve that exact wording so existing automation
// and the bats suite keep matching.
if target.Legacy && strings.Contains(err.Error(), "is not mounted") {
return nil, errors.New("Mount path does not exist.")
}
return nil, err
}
}
return attachments, nil
})
}
// unmountAll removes every attachment on an app, or every attachment for one
@@ -560,19 +553,16 @@ func unmountAll(input CommandUnmountInput) error {
return nil
}
attachments, err := LoadAttachments(input.AppName)
if err != nil {
return err
}
keep := []*Attachment{}
for _, attachment := range attachments {
if attachment.EffectiveProcessType() != input.ProcessType {
keep = append(keep, attachment)
err := UpdateAttachments(input.AppName, func(attachments []*Attachment) ([]*Attachment, error) {
keep := []*Attachment{}
for _, attachment := range attachments {
if attachment.EffectiveProcessType() != input.ProcessType {
keep = append(keep, attachment)
}
}
}
if err := SaveAttachments(input.AppName, keep); err != nil {
return keep, nil
})
if err != nil {
return err
}