fix: serialize concurrent property updates
Concurrent commands for the same app could each read a setting, change it, and write it back, so all of them exited successfully while silently dropping changes made by the others. Updates to app settings are now serialized so every change is kept, and readers never see a partially written value.
This commit is contained in:
42 files changed
+1698
-642
No files matched your search
@@ -105,6 +105,11 @@ func LoadAttachments(appName string) ([]*Attachment, error) {
|
||||
return nil, fmt.Errorf("unable to read storage attachments for %q: %w", appName, err)
|
||||
}
|
||||
|
||||
return decodeAttachments(appName, lines)
|
||||
}
|
||||
|
||||
// decodeAttachments parses the stored attachment lines of an app.
|
||||
func decodeAttachments(appName string, lines []string) ([]*Attachment, error) {
|
||||
attachments := []*Attachment{}
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
@@ -122,15 +127,44 @@ func LoadAttachments(appName string) ([]*Attachment, error) {
|
||||
|
||||
// SaveAttachments overwrites the entire attachment list for an app.
|
||||
func SaveAttachments(appName string, attachments []*Attachment) error {
|
||||
lines, err := encodeAttachments(appName, attachments)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return common.PropertyListWrite(PluginName, appName, AttachmentsProperty, lines)
|
||||
}
|
||||
|
||||
// UpdateAttachments loads the attachment list for an app, passes it to fn and
|
||||
// saves the list fn returns, holding the app's storage property lock across
|
||||
// the read and the write so concurrent mounts and unmounts cannot drop each
|
||||
// other's changes. Nothing is saved when fn returns an error.
|
||||
func UpdateAttachments(appName string, fn func(attachments []*Attachment) ([]*Attachment, error)) error {
|
||||
return common.PropertyListUpdate(PluginName, appName, AttachmentsProperty, func(lines []string) ([]string, error) {
|
||||
attachments, err := decodeAttachments(appName, lines)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
attachments, err = fn(attachments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return encodeAttachments(appName, attachments)
|
||||
})
|
||||
}
|
||||
|
||||
// encodeAttachments serializes attachments into stored attachment lines.
|
||||
func encodeAttachments(appName string, attachments []*Attachment) ([]string, error) {
|
||||
lines := make([]string, 0, len(attachments))
|
||||
for _, attachment := range attachments {
|
||||
data, err := json.Marshal(attachment)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to encode storage attachment for %q: %w", appName, err)
|
||||
return nil, fmt.Errorf("unable to encode storage attachment for %q: %w", appName, err)
|
||||
}
|
||||
lines = append(lines, string(data))
|
||||
}
|
||||
return common.PropertyListWrite(PluginName, appName, AttachmentsProperty, lines)
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// ensureContainerPathFree rejects a candidate whose container path is already
|
||||
@@ -161,26 +195,22 @@ func AddAttachment(appName string, attachment *Attachment) error {
|
||||
return err
|
||||
}
|
||||
|
||||
attachments, err := LoadAttachments(appName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, existing := range attachments {
|
||||
if existing.EntryName == attachment.EntryName &&
|
||||
existing.ContainerPath == attachment.ContainerPath &&
|
||||
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
|
||||
return fmt.Errorf("storage entry %q is already mounted at %q for process type %q on app %q",
|
||||
attachment.EntryName, attachment.ContainerPath, attachment.EffectiveProcessType(), appName)
|
||||
return UpdateAttachments(appName, func(attachments []*Attachment) ([]*Attachment, error) {
|
||||
for _, existing := range attachments {
|
||||
if existing.EntryName == attachment.EntryName &&
|
||||
existing.ContainerPath == attachment.ContainerPath &&
|
||||
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
|
||||
return nil, fmt.Errorf("storage entry %q is already mounted at %q for process type %q on app %q",
|
||||
attachment.EntryName, attachment.ContainerPath, attachment.EffectiveProcessType(), appName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
attachments = append(attachments, attachment)
|
||||
return SaveAttachments(appName, attachments)
|
||||
return append(attachments, attachment), nil
|
||||
})
|
||||
}
|
||||
|
||||
// UpsertAttachment inserts an attachment, or updates the mount-time fields
|
||||
@@ -201,49 +231,43 @@ func UpsertAttachment(appName string, attachment *Attachment) (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
attachments, err := LoadAttachments(appName)
|
||||
created := false
|
||||
err := UpdateAttachments(appName, func(attachments []*Attachment) ([]*Attachment, error) {
|
||||
for _, existing := range attachments {
|
||||
if existing.EntryName == attachment.EntryName &&
|
||||
existing.ContainerPath == attachment.ContainerPath &&
|
||||
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
|
||||
existing.Phases = attachment.Phases
|
||||
existing.Subpath = attachment.Subpath
|
||||
existing.Readonly = attachment.Readonly
|
||||
existing.VolumeOptions = attachment.VolumeOptions
|
||||
existing.VolumeChown = attachment.VolumeChown
|
||||
return attachments, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Only the append branch needs the overlap check: rewriting an existing
|
||||
// tuple in place would otherwise trip over the attachment it is updating.
|
||||
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
created = true
|
||||
return append(attachments, attachment), nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for _, existing := range attachments {
|
||||
if existing.EntryName == attachment.EntryName &&
|
||||
existing.ContainerPath == attachment.ContainerPath &&
|
||||
existing.EffectiveProcessType() == attachment.EffectiveProcessType() {
|
||||
existing.Phases = attachment.Phases
|
||||
existing.Subpath = attachment.Subpath
|
||||
existing.Readonly = attachment.Readonly
|
||||
existing.VolumeOptions = attachment.VolumeOptions
|
||||
existing.VolumeChown = attachment.VolumeChown
|
||||
return false, SaveAttachments(appName, attachments)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the append branch needs the overlap check: rewriting an existing
|
||||
// tuple in place would otherwise trip over the attachment it is updating.
|
||||
if err := ensureContainerPathFree(appName, attachments, attachment); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
attachments = append(attachments, attachment)
|
||||
return true, SaveAttachments(appName, attachments)
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// RemoveAttachment removes the attachment matching the given entry and
|
||||
// optional container path. If containerPath is empty and there is more
|
||||
// than one match, returns an error.
|
||||
func RemoveAttachment(appName string, entryName string, containerPath string) error {
|
||||
attachments, err := LoadAttachments(appName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
keep, err := removeMatchingAttachments(attachments, appName, entryName, containerPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return SaveAttachments(appName, keep)
|
||||
return UpdateAttachments(appName, func(attachments []*Attachment) ([]*Attachment, error) {
|
||||
return removeMatchingAttachments(attachments, appName, entryName, containerPath)
|
||||
})
|
||||
}
|
||||
|
||||
// removeMatchingAttachments returns the given list with the attachments
|
||||
|
||||
@@ -274,32 +274,27 @@ func CommandSet(input CommandSetInput) error {
|
||||
if !EntryExists(input.Name) {
|
||||
return fmt.Errorf("storage entry %q does not exist", input.Name)
|
||||
}
|
||||
entry, err := LoadEntry(input.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
touchesDirectory := false
|
||||
for _, change := range input.Changes {
|
||||
if err := applyPropertyChange(entry, change); err != nil {
|
||||
return err
|
||||
entry, err := UpdateEntry(input.Name, func(entry *Entry) error {
|
||||
for _, change := range input.Changes {
|
||||
if err := applyPropertyChange(entry, change); err != nil {
|
||||
return err
|
||||
}
|
||||
if change.Property == "chown" || change.Property == "mode" {
|
||||
touchesDirectory = true
|
||||
}
|
||||
}
|
||||
if change.Property == "chown" || change.Property == "mode" {
|
||||
touchesDirectory = true
|
||||
|
||||
if input.Annotations != nil {
|
||||
entry.Annotations = input.Annotations
|
||||
}
|
||||
if input.Labels != nil {
|
||||
entry.Labels = input.Labels
|
||||
}
|
||||
}
|
||||
|
||||
if input.Annotations != nil {
|
||||
entry.Annotations = input.Annotations
|
||||
}
|
||||
if input.Labels != nil {
|
||||
entry.Labels = input.Labels
|
||||
}
|
||||
|
||||
if err := entry.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := SaveEntry(entry); err != nil {
|
||||
return entry.Validate()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Only converge the directory when the caller actually asked to change
|
||||
|
||||
@@ -76,32 +76,27 @@ func setEntryMapKey(field entryMapField, name string, key string, value string)
|
||||
return fmt.Errorf("storage entry %q does not exist", name)
|
||||
}
|
||||
|
||||
entry, err := LoadEntry(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
values := field.Get(entry)
|
||||
if value == "" {
|
||||
delete(values, key)
|
||||
common.LogInfo2Quiet(fmt.Sprintf("Unsetting %s %s", field.Singular, key))
|
||||
} else {
|
||||
if values == nil {
|
||||
values = map[string]string{}
|
||||
entry, err := UpdateEntry(name, func(entry *Entry) error {
|
||||
values := field.Get(entry)
|
||||
if value == "" {
|
||||
delete(values, key)
|
||||
common.LogInfo2Quiet(fmt.Sprintf("Unsetting %s %s", field.Singular, key))
|
||||
} else {
|
||||
if values == nil {
|
||||
values = map[string]string{}
|
||||
}
|
||||
values[key] = value
|
||||
common.LogInfo2Quiet(fmt.Sprintf("Setting %s %s to %s", field.Singular, key, value))
|
||||
}
|
||||
values[key] = value
|
||||
common.LogInfo2Quiet(fmt.Sprintf("Setting %s %s to %s", field.Singular, key, value))
|
||||
}
|
||||
if len(values) == 0 {
|
||||
// Drop the empty map so the omitempty tag keeps it out of the JSON.
|
||||
values = nil
|
||||
}
|
||||
field.Set(entry, values)
|
||||
if len(values) == 0 {
|
||||
// Drop the empty map so the omitempty tag keeps it out of the JSON.
|
||||
values = nil
|
||||
}
|
||||
field.Set(entry, values)
|
||||
|
||||
if err := entry.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := SaveEntry(entry); err != nil {
|
||||
return entry.Validate()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestAddAttachmentConcurrent(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
withTempLibRoot(t)
|
||||
|
||||
const count = 20
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, count)
|
||||
for i := 0; i < count; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
errs[i] = AddAttachment("demo", &Attachment{
|
||||
EntryName: fmt.Sprintf("demo-data-%d", i),
|
||||
ContainerPath: fmt.Sprintf("/data/%d", i),
|
||||
Phases: []string{PhaseDeploy, PhaseRun},
|
||||
ProcessType: DefaultProcessType,
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
Expect(errors.Join(errs...)).NotTo(HaveOccurred())
|
||||
|
||||
attachments, err := LoadAttachments("demo")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(attachments).To(HaveLen(count))
|
||||
}
|
||||
|
||||
func TestUpdateEntryConcurrent(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
withTempLibRoot(t)
|
||||
|
||||
Expect(SaveEntry(&Entry{
|
||||
Name: "demo-data",
|
||||
Scheduler: SchedulerK3s,
|
||||
Size: "2Gi",
|
||||
})).To(Succeed())
|
||||
|
||||
const count = 20
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, count)
|
||||
for i := 0; i < count; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
_, errs[i] = UpdateEntry("demo-data", func(entry *Entry) error {
|
||||
if entry.Labels == nil {
|
||||
entry.Labels = map[string]string{}
|
||||
}
|
||||
entry.Labels[fmt.Sprintf("label-%d", i)] = "value"
|
||||
return nil
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
Expect(errors.Join(errs...)).NotTo(HaveOccurred())
|
||||
|
||||
entry, err := LoadEntry("demo-data")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(entry.Labels).To(HaveLen(count))
|
||||
}
|
||||
|
||||
func TestUpdateEntryErrorLeavesEntryUnchanged(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
withTempLibRoot(t)
|
||||
|
||||
Expect(SaveEntry(&Entry{
|
||||
Name: "demo-data",
|
||||
Scheduler: SchedulerK3s,
|
||||
Size: "2Gi",
|
||||
})).To(Succeed())
|
||||
|
||||
_, err := UpdateEntry("demo-data", func(entry *Entry) error {
|
||||
entry.Size = "5Gi"
|
||||
return errors.New("boom")
|
||||
})
|
||||
Expect(err).To(MatchError("boom"))
|
||||
|
||||
entry, err := LoadEntry("demo-data")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(entry.Size).To(Equal("2Gi"))
|
||||
}
|
||||
@@ -67,7 +67,7 @@ var (
|
||||
)
|
||||
|
||||
// Entry is the source of truth for a storage volume. One file per entry
|
||||
// lives at $DOKKU_LIB_ROOT/config/storage/entries/<name>.json.
|
||||
// lives at $DOKKU_LIB_ROOT/data/storage-registry/entries/<name>.json.
|
||||
type Entry struct {
|
||||
Name string `json:"name"`
|
||||
Scheduler string `json:"scheduler"`
|
||||
@@ -162,6 +162,39 @@ func SaveEntry(entry *Entry) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateEntry loads a registered entry, passes it to fn and saves it, holding
|
||||
// the entries directory lock across the read and the write so concurrent
|
||||
// edits to one entry cannot drop each other's changes. Nothing is saved when
|
||||
// fn returns an error. The saved entry is returned.
|
||||
func UpdateEntry(name string, fn func(entry *Entry) error) (*Entry, error) {
|
||||
if err := EnsureEntriesDirectory(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var updated *Entry
|
||||
err := common.LockDirectory(EntriesDirectory(), func() error {
|
||||
entry, err := LoadEntry(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := fn(entry); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := SaveEntry(entry); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updated = entry
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// DeleteEntry removes the on-disk record for an entry.
|
||||
func DeleteEntry(name string) error {
|
||||
if err := os.Remove(entryPath(name)); err != nil && !os.IsNotExist(err) {
|
||||
|
||||
@@ -217,19 +217,18 @@ func migrateMount(appName string, processType string, mount string, phases []str
|
||||
VolumeOptions: parsed.VolumeOptions,
|
||||
}
|
||||
|
||||
existing, err := LoadAttachments(appName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A named scope whose mount the default scope already covers needs no
|
||||
// attachment of its own - the default one reaches that process anyway, and
|
||||
// recording both would manufacture two -v flags for one container path.
|
||||
// The docker-options line is still drained so it stops being emitted twice.
|
||||
if !attachmentExists(existing, attachment) && !coveredByDefaultScope(existing, attachment) {
|
||||
existing = append(existing, attachment)
|
||||
if err := SaveAttachments(appName, existing); err != nil {
|
||||
return err
|
||||
err := UpdateAttachments(appName, func(existing []*Attachment) ([]*Attachment, error) {
|
||||
if attachmentExists(existing, attachment) || coveredByDefaultScope(existing, attachment) {
|
||||
return existing, nil
|
||||
}
|
||||
return append(existing, attachment), nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mountedPhases := []string{}
|
||||
|
||||
@@ -390,34 +390,30 @@ func replaceMounts(input CommandMountInput) error {
|
||||
}
|
||||
}
|
||||
|
||||
existing, err := LoadAttachments(input.AppName)
|
||||
err := UpdateAttachments(input.AppName, func(existing []*Attachment) ([]*Attachment, error) {
|
||||
retained := []*Attachment{}
|
||||
for _, attachment := range existing {
|
||||
if attachment.EffectiveProcessType() != processType {
|
||||
retained = append(retained, attachment)
|
||||
}
|
||||
}
|
||||
|
||||
// The declared set is already unique on container path within its own
|
||||
// scope; what is left to check is the scopes this call does not replace,
|
||||
// so a declared path cannot collide with one held by another process type.
|
||||
// Checking before the write keeps the all-or-nothing contract.
|
||||
for _, attachment := range declared {
|
||||
if err := ensureContainerPathFree(input.AppName, retained, attachment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return append(retained, declared...), nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
retained := []*Attachment{}
|
||||
for _, attachment := range existing {
|
||||
if attachment.EffectiveProcessType() != processType {
|
||||
retained = append(retained, attachment)
|
||||
}
|
||||
}
|
||||
|
||||
// The declared set is already unique on container path within its own
|
||||
// scope; what is left to check is the scopes this call does not replace,
|
||||
// so a declared path cannot collide with one held by another process type.
|
||||
// Checking before the write keeps the all-or-nothing contract.
|
||||
for _, attachment := range declared {
|
||||
if err := ensureContainerPathFree(input.AppName, retained, attachment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
attachments := append(retained, declared...)
|
||||
|
||||
if err := SaveAttachments(input.AppName, attachments); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
common.LogInfo1(fmt.Sprintf("Storage mounts for process type %s replaced on %s", processType, input.AppName))
|
||||
return nil
|
||||
}
|
||||
@@ -510,30 +506,27 @@ func resolveUnmountTarget(mount string, containerDir string) (unmountTarget, err
|
||||
// written, so an argument naming a mount the app does not have leaves the rest
|
||||
// of the set in place rather than removing a prefix of it.
|
||||
func unmountMounts(input CommandUnmountInput) error {
|
||||
attachments, err := LoadAttachments(input.AppName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, mount := range input.Mounts {
|
||||
target, err := resolveUnmountTarget(mount, input.ContainerDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attachments, err = removeMatchingAttachments(attachments, input.AppName, target.EntryName, target.ContainerPath)
|
||||
if err != nil {
|
||||
// The legacy form historically said "Mount path does not
|
||||
// exist." - preserve that exact wording so existing automation
|
||||
// and the bats suite keep matching.
|
||||
if target.Legacy && strings.Contains(err.Error(), "is not mounted") {
|
||||
return errors.New("Mount path does not exist.")
|
||||
return UpdateAttachments(input.AppName, func(attachments []*Attachment) ([]*Attachment, error) {
|
||||
for _, mount := range input.Mounts {
|
||||
target, err := resolveUnmountTarget(mount, input.ContainerDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return SaveAttachments(input.AppName, attachments)
|
||||
attachments, err = removeMatchingAttachments(attachments, input.AppName, target.EntryName, target.ContainerPath)
|
||||
if err != nil {
|
||||
// The legacy form historically said "Mount path does not
|
||||
// exist." - preserve that exact wording so existing automation
|
||||
// and the bats suite keep matching.
|
||||
if target.Legacy && strings.Contains(err.Error(), "is not mounted") {
|
||||
return nil, errors.New("Mount path does not exist.")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return attachments, nil
|
||||
})
|
||||
}
|
||||
|
||||
// unmountAll removes every attachment on an app, or every attachment for one
|
||||
@@ -560,19 +553,16 @@ func unmountAll(input CommandUnmountInput) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
attachments, err := LoadAttachments(input.AppName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
keep := []*Attachment{}
|
||||
for _, attachment := range attachments {
|
||||
if attachment.EffectiveProcessType() != input.ProcessType {
|
||||
keep = append(keep, attachment)
|
||||
err := UpdateAttachments(input.AppName, func(attachments []*Attachment) ([]*Attachment, error) {
|
||||
keep := []*Attachment{}
|
||||
for _, attachment := range attachments {
|
||||
if attachment.EffectiveProcessType() != input.ProcessType {
|
||||
keep = append(keep, attachment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := SaveAttachments(input.AppName, keep); err != nil {
|
||||
return keep, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user