diff --git a/plugins/storage/Makefile b/plugins/storage/Makefile index c719dde7d..7a6b61e35 100644 --- a/plugins/storage/Makefile +++ b/plugins/storage/Makefile @@ -1,6 +1,6 @@ GOARCH ?= amd64 SUBCOMMANDS = subcommands/default subcommands/create subcommands/destroy subcommands/ensure-directory subcommands/info subcommands/list subcommands/list-entries subcommands/mount subcommands/report subcommands/unmount -TRIGGERS = triggers/install triggers/storage-list +TRIGGERS = triggers/install triggers/storage-list triggers/storage-app-mounts triggers/docker-args-deploy triggers/docker-args-run triggers/post-delete triggers/post-app-clone-setup triggers/post-app-rename-setup BUILD = commands subcommands triggers PLUGIN_NAME = storage diff --git a/plugins/storage/migrate.go b/plugins/storage/migrate.go new file mode 100644 index 000000000..088ad4d83 --- /dev/null +++ b/plugins/storage/migrate.go @@ -0,0 +1,190 @@ +package storage + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/dokku/dokku/plugins/common" + dockeroptions "github.com/dokku/dokku/plugins/docker-options" +) + +// migrationFlagDir contains per-app flag files written after a successful +// per-app migration; presence means the app's legacy -v lines have been +// drained into the attachment store. +func migrationFlagDir() string { + root := common.GetenvWithDefault("DOKKU_LIB_ROOT", "/var/lib/dokku") + return filepath.Join(root, "config", "storage", ".migrated") +} + +func migrationFlagFile(appName string) string { + return filepath.Join(migrationFlagDir(), appName) +} + +// MigrateLegacyMounts walks every app and converts its legacy `-v` +// docker-options entries into named storage entries plus attachments. +// Idempotent: per-app flag files short-circuit re-runs. +func MigrateLegacyMounts() error { + if err := os.MkdirAll(migrationFlagDir(), 0755); err != nil { + return fmt.Errorf("unable to create migration flag dir: %w", err) + } + + apps, err := common.DokkuApps() + if err != nil { + if errors.Is(err, common.NoAppsExist) { + return nil + } + return err + } + + for _, app := range apps { + if _, err := os.Stat(migrationFlagFile(app)); err == nil { + continue + } + if err := migrateApp(app); err != nil { + return fmt.Errorf("storage migration failed for app %q: %w", app, err) + } + } + return nil +} + +// migrateApp performs the per-app migration. Order is intentional: write +// the entry, write the attachment, drain the docker-options line. Any +// step failing aborts before drain, so the original behavior is +// preserved on partial failure. +func migrateApp(appName string) error { + deployLines, err := dockeroptions.GetDockerOptionsForPhase(appName, PhaseDeploy) + if err != nil { + return err + } + runLines, err := dockeroptions.GetDockerOptionsForPhase(appName, PhaseRun) + if err != nil { + return err + } + + deployMounts := filterMountLines(deployLines) + runMounts := filterMountLines(runLines) + + // Group identical mount strings across phases. + phaseMap := map[string][]string{} + for _, mount := range deployMounts { + phaseMap[mount] = appendUnique(phaseMap[mount], PhaseDeploy) + } + for _, mount := range runMounts { + phaseMap[mount] = appendUnique(phaseMap[mount], PhaseRun) + } + + mounts := make([]string, 0, len(phaseMap)) + for mount := range phaseMap { + mounts = append(mounts, mount) + } + sort.Strings(mounts) + + for _, mount := range mounts { + if err := migrateMount(appName, mount, phaseMap[mount]); err != nil { + return err + } + } + + return touchMigrationFlag(appName) +} + +func migrateMount(appName string, mount string, phases []string) error { + parsed := ParseMountPath(mount) + entry := LegacyMountToEntry(mount) + + if EntryExists(entry.Name) { + existing, err := LoadEntry(entry.Name) + if err != nil { + return err + } + if existing.HostPath != entry.HostPath || existing.Scheduler != entry.Scheduler { + return fmt.Errorf("legacy entry %q already exists with conflicting fields", entry.Name) + } + } else { + if err := SaveEntry(entry); err != nil { + return err + } + } + + containerPath := parsed.ContainerPath + if containerPath == "" { + // Defensive: legacy mounts must have a container path; skip if not. + return nil + } + + attachment := &Attachment{ + EntryName: entry.Name, + ContainerPath: containerPath, + Phases: phases, + ProcessType: DefaultProcessType, + } + if parsed.VolumeOptions != "" { + if parsed.VolumeOptions == "ro" { + attachment.Readonly = true + } else { + attachment.VolumeOptions = parsed.VolumeOptions + } + } + + existing, err := LoadAttachments(appName) + if err != nil { + return err + } + if !attachmentExists(existing, attachment) { + existing = append(existing, attachment) + if err := SaveAttachments(appName, existing); err != nil { + return err + } + } + + mountedPhases := []string{} + for _, phase := range phases { + mountedPhases = append(mountedPhases, phase) + } + if err := dockeroptions.RemoveDockerOptionFromPhases(appName, mountedPhases, fmt.Sprintf("-v %s", mount)); err != nil { + return err + } + return nil +} + +func touchMigrationFlag(appName string) error { + f, err := os.Create(migrationFlagFile(appName)) + if err != nil { + return err + } + return f.Close() +} + +func filterMountLines(lines []string) []string { + out := []string{} + for _, line := range lines { + if strings.HasPrefix(line, "-v ") { + out = append(out, strings.TrimPrefix(line, "-v ")) + } + } + return out +} + +func appendUnique(slice []string, value string) []string { + for _, existing := range slice { + if existing == value { + return slice + } + } + return append(slice, value) +} + +func attachmentExists(attachments []*Attachment, candidate *Attachment) bool { + for _, existing := range attachments { + if existing.EntryName == candidate.EntryName && + existing.ContainerPath == candidate.ContainerPath && + existing.ProcessType == candidate.ProcessType { + return true + } + } + return false +} diff --git a/plugins/storage/src/subcommands/subcommands.go b/plugins/storage/src/subcommands/subcommands.go index 86ce98ecf..52d9a46bb 100644 --- a/plugins/storage/src/subcommands/subcommands.go +++ b/plugins/storage/src/subcommands/subcommands.go @@ -86,10 +86,23 @@ func main() { err = storage.CommandListEntries(*scheduler, *format) case "mount": args := flag.NewFlagSet("storage:mount", flag.ExitOnError) + containerDir := args.String("container-dir", "", "--container-dir: container path (named-entry form)") + phases := args.StringSlice("phase", nil, "--phase: phase to mount in (deploy, run; default both)") + processType := args.String("process-type", "", "--process-type: process type to mount for") + subpath := args.String("volume-subpath", "", "--volume-subpath: subpath within the entry") + readonly := args.Bool("volume-readonly", false, "--volume-readonly: mount the volume read-only") + volumeChown := args.String("volume-chown", "", "--volume-chown: chown option applied at mount time") args.Parse(os.Args[2:]) - appName := args.Arg(0) - mountPath := args.Arg(1) - err = storage.CommandMount(appName, mountPath) + err = storage.CommandMount(storage.CommandMountInput{ + AppName: args.Arg(0), + NameOrPath: args.Arg(1), + ContainerDir: *containerDir, + Phases: *phases, + ProcessType: *processType, + Subpath: *subpath, + Readonly: *readonly, + VolumeChown: *volumeChown, + }) case "report": args := flag.NewFlagSet("storage:report", flag.ExitOnError) format := args.String("format", "stdout", "format: [ stdout | json ]") @@ -104,10 +117,13 @@ func main() { } case "unmount": args := flag.NewFlagSet("storage:unmount", flag.ExitOnError) + containerDir := args.String("container-dir", "", "--container-dir: container path (named-entry form, disambiguates duplicates)") args.Parse(os.Args[2:]) - appName := args.Arg(0) - mountPath := args.Arg(1) - err = storage.CommandUnmount(appName, mountPath) + err = storage.CommandUnmount(storage.CommandUnmountInput{ + AppName: args.Arg(0), + NameOrPath: args.Arg(1), + ContainerDir: *containerDir, + }) default: err = fmt.Errorf("Invalid plugin subcommand call: %s", subcommand) } diff --git a/plugins/storage/src/triggers/triggers.go b/plugins/storage/src/triggers/triggers.go index 3d2dd3ebf..565891290 100644 --- a/plugins/storage/src/triggers/triggers.go +++ b/plugins/storage/src/triggers/triggers.go @@ -24,6 +24,27 @@ func main() { phase := flag.Arg(1) format := flag.Arg(2) err = storage.TriggerStorageList(appName, phase, format) + case "storage-app-mounts": + appName := flag.Arg(0) + phase := flag.Arg(1) + err = storage.TriggerStorageAppMounts(appName, phase) + case "docker-args-deploy": + appName := flag.Arg(0) + err = storage.TriggerDockerArgs(appName, storage.PhaseDeploy) + case "docker-args-run": + appName := flag.Arg(0) + err = storage.TriggerDockerArgs(appName, storage.PhaseRun) + case "post-delete": + appName := flag.Arg(0) + err = storage.TriggerPostDelete(appName) + case "post-app-clone-setup": + oldName := flag.Arg(0) + newName := flag.Arg(1) + err = storage.TriggerPostAppCloneSetup(oldName, newName) + case "post-app-rename-setup": + oldName := flag.Arg(0) + newName := flag.Arg(1) + err = storage.TriggerPostAppRenameSetup(oldName, newName) default: err = fmt.Errorf("Invalid plugin trigger call: %s", trigger) } diff --git a/plugins/storage/subcommands.go b/plugins/storage/subcommands.go index b84575b84..9b3eb5824 100644 --- a/plugins/storage/subcommands.go +++ b/plugins/storage/subcommands.go @@ -125,38 +125,102 @@ func isUserNamespacesEnabled() (bool, error) { return strings.TrimSpace(result.StdoutContents()) == "true", nil } -// CommandMount creates a new bind mount for an app -func CommandMount(appName string, mountPath string) error { - if err := common.VerifyAppName(appName); err != nil { - return err - } - - if err := VerifyPaths(mountPath); err != nil { - return err - } - - if CheckIfPathExists(appName, mountPath, MountPhases) { - return errors.New("Mount path already exists.") - } - - return dockeroptions.AddDockerOptionToPhases(appName, MountPhases, fmt.Sprintf("-v %s", mountPath)) +// CommandMountInput captures the optional flags accepted by storage:mount +// when the second argument is a named entry rather than a colon-form path. +type CommandMountInput struct { + AppName string + NameOrPath string + ContainerDir string + Phases []string + ProcessType string + Subpath string + Readonly bool + VolumeOptions string + VolumeChown string } -// CommandUnmount removes an existing bind mount from an app -func CommandUnmount(appName string, mountPath string) error { - if err := common.VerifyAppName(appName); err != nil { +// CommandMount creates a new bind mount for an app. The second positional +// argument may be either a legacy host:container[:options] string (kept +// for back-compat on docker-local) or a registered storage entry name. +func CommandMount(input CommandMountInput) error { + if err := common.VerifyAppName(input.AppName); err != nil { return err } - if err := VerifyPaths(mountPath); err != nil { + // Legacy colon form: keep writing into docker-options so existing + // users see no behavior change. + if strings.Contains(input.NameOrPath, ":") { + if err := VerifyPaths(input.NameOrPath); err != nil { + return err + } + if CheckIfPathExists(input.AppName, input.NameOrPath, MountPhases) { + return errors.New("Mount path already exists.") + } + return dockeroptions.AddDockerOptionToPhases(input.AppName, MountPhases, fmt.Sprintf("-v %s", input.NameOrPath)) + } + + // Named-entry form: persist as an attachment. + if !EntryExists(input.NameOrPath) { + return fmt.Errorf("storage entry %q does not exist; create it first with `dokku storage:create`", input.NameOrPath) + } + if input.ContainerDir == "" { + return errors.New("--container-dir is required when mounting a named storage entry") + } + + entry, err := LoadEntry(input.NameOrPath) + if err != nil { return err } - if !CheckIfPathExists(appName, mountPath, MountPhases) { - return errors.New("Mount path does not exist.") + phases := input.Phases + if len(phases) == 0 { + phases = []string{PhaseDeploy, PhaseRun} } - return dockeroptions.RemoveDockerOptionFromPhases(appName, MountPhases, fmt.Sprintf("-v %s", mountPath)) + processType := input.ProcessType + if processType == "" { + processType = DefaultProcessType + } + + attachment := &Attachment{ + EntryName: entry.Name, + ContainerPath: input.ContainerDir, + Phases: phases, + ProcessType: processType, + Subpath: input.Subpath, + Readonly: input.Readonly, + VolumeOptions: input.VolumeOptions, + VolumeChown: input.VolumeChown, + } + + return AddAttachment(input.AppName, attachment) +} + +// CommandUnmountInput captures the flags accepted by storage:unmount when +// the second argument is a named entry. +type CommandUnmountInput struct { + AppName string + NameOrPath string + ContainerDir string +} + +// CommandUnmount removes an existing bind mount from an app. +func CommandUnmount(input CommandUnmountInput) error { + if err := common.VerifyAppName(input.AppName); err != nil { + return err + } + + if strings.Contains(input.NameOrPath, ":") { + if err := VerifyPaths(input.NameOrPath); err != nil { + return err + } + if !CheckIfPathExists(input.AppName, input.NameOrPath, MountPhases) { + return errors.New("Mount path does not exist.") + } + return dockeroptions.RemoveDockerOptionFromPhases(input.AppName, MountPhases, fmt.Sprintf("-v %s", input.NameOrPath)) + } + + return RemoveAttachment(input.AppName, input.NameOrPath, input.ContainerDir) } // CommandList lists all bind mounts for an app diff --git a/plugins/storage/triggers.go b/plugins/storage/triggers.go index ac2c4d4e0..389783e42 100644 --- a/plugins/storage/triggers.go +++ b/plugins/storage/triggers.go @@ -6,11 +6,13 @@ import ( "os" "path/filepath" "runtime" + "strings" "github.com/dokku/dokku/plugins/common" ) -// TriggerInstall sets up the storage plugin on installation +// TriggerInstall sets up the storage plugin on installation and runs +// the bulk legacy-mount migration once per upgrade. func TriggerInstall() error { storageDir := GetStorageDirectory() @@ -25,6 +27,14 @@ func TriggerInstall() error { return fmt.Errorf("Unable to set storage directory permissions: %s", err.Error()) } + if err := EnsureEntriesDirectory(); err != nil { + return err + } + + if err := MigrateLegacyMounts(); err != nil { + return fmt.Errorf("storage migration failed: %w", err) + } + distro := detectDistro() if distro == "" { return nil @@ -44,6 +54,44 @@ func TriggerInstall() error { return nil } +// TriggerPostDelete clears any attachment list for an app that's being +// destroyed. Entries are global and survive app deletion. +func TriggerPostDelete(appName string) error { + if appName == "" { + return nil + } + return common.PropertyListWrite(PluginName, appName, AttachmentsProperty, []string{}) +} + +// TriggerPostAppCloneSetup copies attachments from the source app to the +// cloned app. Entries are global so no entry-level work is needed. +func TriggerPostAppCloneSetup(oldName string, newName string) error { + if oldName == "" || newName == "" { + return nil + } + attachments, err := LoadAttachments(oldName) + if err != nil { + return err + } + return SaveAttachments(newName, attachments) +} + +// TriggerPostAppRenameSetup moves attachments from the old name to the +// new name and clears the old. +func TriggerPostAppRenameSetup(oldName string, newName string) error { + if oldName == "" || newName == "" { + return nil + } + attachments, err := LoadAttachments(oldName) + if err != nil { + return err + } + if err := SaveAttachments(newName, attachments); err != nil { + return err + } + return common.PropertyListWrite(PluginName, oldName, AttachmentsProperty, []string{}) +} + // detectDistro returns the Linux distribution name func detectDistro() string { if runtime.GOOS != "linux" { @@ -91,3 +139,87 @@ func TriggerStorageList(appName string, phase string, format string) error { return nil } + +// AppMountPair pairs an Attachment with the Entry it references; consumed +// by scheduler plugins via the storage-app-mounts trigger. +type AppMountPair struct { + Entry *Entry `json:"entry"` + Attachment *Attachment `json:"attachment"` +} + +// TriggerStorageAppMounts emits the (entry, attachment) pairs an app has +// for the given phase, in JSON. Schedulers consume this at deploy time. +func TriggerStorageAppMounts(appName string, phase string) error { + if phase == "" { + phase = PhaseDeploy + } + attachments, err := AttachmentsForPhase(appName, phase) + if err != nil { + return err + } + + pairs := []AppMountPair{} + for _, attachment := range attachments { + entry, err := LoadEntry(attachment.EntryName) + if err != nil { + return fmt.Errorf("attachment on %q references missing entry %q: %w", appName, attachment.EntryName, err) + } + pairs = append(pairs, AppMountPair{Entry: entry, Attachment: attachment}) + } + + output, err := json.Marshal(pairs) + if err != nil { + return err + } + fmt.Println(string(output)) + return nil +} + +// TriggerDockerArgs emits `-v` flags for each docker-local attachment in +// the requested phase. Plugn concatenates this with docker-options' +// equivalent trigger output, so docker-local apps continue to receive +// their bind mounts through the standard pipeline. +func TriggerDockerArgs(appName string, phase string) error { + attachments, err := AttachmentsForPhase(appName, phase) + if err != nil { + return err + } + + for _, attachment := range attachments { + entry, err := LoadEntry(attachment.EntryName) + if err != nil { + return fmt.Errorf("attachment on %q references missing entry %q: %w", appName, attachment.EntryName, err) + } + if entry.Scheduler != SchedulerDockerLocal { + continue + } + flag := buildDockerVFlag(entry, attachment) + if flag == "" { + continue + } + fmt.Printf(" %s", flag) + } + return nil +} + +// buildDockerVFlag formats the Docker -v argument for a docker-local +// attachment. +func buildDockerVFlag(entry *Entry, attachment *Attachment) string { + host := entry.HostPath + container := attachment.ContainerPath + if host == "" || container == "" { + return "" + } + options := []string{} + if attachment.Readonly { + options = append(options, "ro") + } + if attachment.VolumeOptions != "" { + options = append(options, attachment.VolumeOptions) + } + flag := fmt.Sprintf("-v %s:%s", host, container) + if len(options) > 0 { + flag = fmt.Sprintf("%s:%s", flag, strings.Join(options, ",")) + } + return flag +}