feat: delegate storage:exec to scheduler plugins, harden the flow

Moves the actual exec out of the storage plugin and into a new scheduler-storage-exec plugn trigger. scheduler-docker-local does docker run with TTY-aware -it/-i selection and --user derived from entry.Chown; scheduler-k3s creates a throwaway Pod via the kubernetes API, waits for it to reach Running with structured error reporting (ImagePullBackOff and friends are surfaced from the container status verbatim, no kubectl involvement), execs the user command via the existing SPDY plumbing in k8s.go, and deletes the Pod on the way out. (Entry).Validate now accepts either an absolute path or a docker named-volume token for docker-local entries so the migration synthesizer's named-volume legacy entries work cleanly. storage:exec gains --as-user for one-off uid overrides, propagates the underlying tool's exit code via os.Exit, and detects TTY/interactive mode from os.Stdin so non-interactive scripted use no longer trips over docker's input-device-is-not-a-tty error.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-04-29 23:29:30 -04:00
1 parent d75228f165
commit 58042b9330
17 files changed
+790 -72

No files matched your search

+1 -1
View File
@@ -1,5 +1,5 @@
SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/storage-create triggers/storage-destroy triggers/storage-status
TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec
BUILD = commands subcommands triggers
PLUGIN_NAME = scheduler-k3s
@@ -19,6 +19,9 @@ func main() {
parts := strings.Split(os.Args[0], "/")
trigger := parts[len(parts)-1]
podIdentifier := flag.String("container-id", "", "--container-id: A pod identifier")
storageExecInteractive := flag.Bool("interactive", false, "--interactive: stdin is open (storage:exec)")
storageExecTty := flag.Bool("tty", false, "--tty: stdin is a terminal (storage:exec)")
storageExecAsUser := flag.String("as-user", "", "--as-user: numeric uid override (storage:exec)")
flag.Parse()
var err error
@@ -168,6 +171,27 @@ func main() {
case "storage-status":
entryName := flag.Arg(0)
err = scheduler_k3s.TriggerStorageStatus(context.Background(), entryName)
case "scheduler-storage-exec":
positional := flag.Args()
if len(positional) < 3 {
err = fmt.Errorf("scheduler-storage-exec requires <scheduler> <entry-name> <image>")
break
}
schedulerName := positional[0]
entryName := positional[1]
image := positional[2]
var cmd []string
if len(positional) > 3 {
cmd = positional[3:]
}
err = scheduler_k3s.TriggerSchedulerStorageExec(context.Background(), schedulerName, scheduler_k3s.StorageExecInput{
EntryName: entryName,
Image: image,
Interactive: *storageExecInteractive,
Tty: *storageExecTty,
AsUser: *storageExecAsUser,
Command: cmd,
})
default:
err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
}
+236
View File
@@ -0,0 +1,236 @@
package scheduler_k3s
import (
"context"
"errors"
"fmt"
"os"
"strconv"
"time"
"github.com/dokku/dokku/plugins/common"
"github.com/dokku/dokku/plugins/storage"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilexec "k8s.io/client-go/util/exec"
)
// StorageExecInput captures the inputs forwarded by the storage plugin
// over the scheduler-storage-exec trigger.
type StorageExecInput struct {
EntryName string
Image string
Interactive bool
Tty bool
AsUser string
Command []string
}
// TriggerSchedulerStorageExec runs an interactive or non-interactive
// command against a k3s storage entry by creating a throwaway Pod that
// mounts the PVC at /data, exec-ing the user's command via the
// kubernetes API, and deleting the Pod afterwards. Errors at every
// stage (PVC missing, image pull failure, etc.) come through the
// structured Kubernetes API objects rather than kubectl-shaped strings.
func TriggerSchedulerStorageExec(ctx context.Context, scheduler string, input StorageExecInput) error {
if scheduler != storage.SchedulerK3s {
return nil
}
if !storage.EntryExists(input.EntryName) {
return fmt.Errorf("storage entry %q does not exist", input.EntryName)
}
entry, err := storage.LoadEntry(input.EntryName)
if err != nil {
return err
}
if entry.Scheduler != storage.SchedulerK3s {
return fmt.Errorf("storage entry %q has scheduler %q, not k3s", entry.Name, entry.Scheduler)
}
if err := isKubernetesAvailable(); err != nil {
return fmt.Errorf("kubernetes not available: %w", err)
}
clientset, err := NewKubernetesClient()
if err != nil {
return err
}
namespace := entry.Namespace
if namespace == "" {
namespace = "default"
}
if _, err := clientset.Client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, entry.Name, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
return fmt.Errorf("PVC %s not found in namespace %s", entry.Name, namespace)
}
return fmt.Errorf("error reading PVC %s in namespace %s: %w", entry.Name, namespace, err)
}
podName := fmt.Sprintf("dokku-storage-exec-%s-%d", entry.Name, time.Now().UnixNano()/int64(time.Millisecond)%1000000)
pod, err := buildStoragePodSpec(podName, namespace, entry, input)
if err != nil {
return err
}
common.LogVerboseQuiet(fmt.Sprintf("Creating exec pod %s/%s", namespace, podName))
if _, err := clientset.Client.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
return fmt.Errorf("error creating exec pod %s/%s: %w", namespace, podName, err)
}
defer func() {
bg := context.Background()
gracePeriod := int64(0)
_ = clientset.Client.CoreV1().Pods(namespace).Delete(bg, podName, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod})
}()
if err := waitForPodRunning(ctx, clientset, namespace, podName, 60*time.Second); err != nil {
return err
}
command := input.Command
if len(command) == 0 {
command = []string{"sh", "-c", "command -v bash >/dev/null 2>&1 && exec bash || exec sh"}
}
execErr := clientset.ExecCommand(ctx, ExecCommandInput{
Name: podName,
Namespace: namespace,
ContainerName: "exec",
Command: command,
})
if execErr == nil {
return nil
}
var coded utilexec.CodeExitError
if errors.As(execErr, &coded) {
os.Exit(coded.Code)
}
return execErr
}
// buildStoragePodSpec assembles the throwaway Pod that mounts the PVC
// for storage:exec. Split out so the unit test can verify the spec
// without hitting a cluster.
func buildStoragePodSpec(name string, namespace string, entry *storage.Entry, input StorageExecInput) (*corev1.Pod, error) {
uid, err := resolveStorageExecUID(entry.Chown, input.AsUser)
if err != nil {
return nil, err
}
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: map[string]string{
"app.kubernetes.io/managed-by": "dokku",
"dokku.com/storage-entry": entry.Name,
"dokku.com/purpose": "storage-exec",
},
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
Volumes: []corev1.Volume{
{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: entry.Name,
},
},
},
},
Containers: []corev1.Container{
{
Name: "exec",
Image: input.Image,
Command: []string{"sleep", "infinity"},
Stdin: input.Interactive,
TTY: input.Tty,
VolumeMounts: []corev1.VolumeMount{
{Name: "data", MountPath: "/data"},
},
},
},
},
}
if uid != nil {
pod.Spec.SecurityContext = &corev1.PodSecurityContext{
RunAsUser: uid,
RunAsGroup: uid,
FSGroup: uid,
}
}
return pod, nil
}
// resolveStorageExecUID picks the UID to run the exec Pod as. Returns
// nil to mean "leave the image's default user in place".
func resolveStorageExecUID(chown string, asUser string) (*int64, error) {
if asUser != "" {
n, err := strconv.ParseInt(asUser, 10, 64)
if err != nil {
return nil, fmt.Errorf("--as-user must be a numeric uid, got %q", asUser)
}
return &n, nil
}
if chown == "" || chown == "false" {
return nil, nil
}
uidStr, err := storage.ResolveChownID(chown)
if err != nil {
return nil, err
}
if uidStr == "" || uidStr == "false" {
return nil, nil
}
n, err := strconv.ParseInt(uidStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("could not parse resolved chown UID %q: %w", uidStr, err)
}
return &n, nil
}
// waitForPodRunning polls the Pod until its phase is Running, surfacing
// the structured Kubernetes API state on failure (image pull failures,
// scheduling problems, etc.) rather than a generic timeout.
func waitForPodRunning(ctx context.Context, clientset KubernetesClient, namespace, name string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for {
pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("error reading exec pod %s/%s: %w", namespace, name, err)
}
switch pod.Status.Phase {
case corev1.PodRunning:
return nil
case corev1.PodFailed, corev1.PodSucceeded:
return fmt.Errorf("exec pod %s/%s reached terminal phase %s before exec could attach", namespace, name, pod.Status.Phase)
}
for _, cs := range pod.Status.ContainerStatuses {
if cs.State.Waiting == nil {
continue
}
reason := cs.State.Waiting.Reason
switch reason {
case "ImagePullBackOff", "ErrImagePull", "CreateContainerConfigError", "InvalidImageName":
return fmt.Errorf("exec pod %s/%s container %q failed to start: %s: %s", namespace, name, cs.Name, reason, cs.State.Waiting.Message)
}
}
if time.Now().After(deadline) {
return fmt.Errorf("exec pod %s/%s did not reach Running within %s (current phase: %s)", namespace, name, timeout, pod.Status.Phase)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
}
}
+103
View File
@@ -0,0 +1,103 @@
package scheduler_k3s
import (
"testing"
"github.com/dokku/dokku/plugins/storage"
)
func TestBuildStoragePodSpec(t *testing.T) {
entry := &storage.Entry{
Name: "demo-data",
Scheduler: storage.SchedulerK3s,
Namespace: "dokku",
}
pod, err := buildStoragePodSpec("dokku-storage-exec-demo-data-1", "dokku", entry, StorageExecInput{
Image: "alpine:3",
Interactive: true,
Tty: true,
Command: []string{"sh"},
})
if err != nil {
t.Fatalf("buildStoragePodSpec err: %v", err)
}
if pod.Spec.RestartPolicy != "Never" {
t.Fatalf("expected RestartPolicy=Never, got %s", pod.Spec.RestartPolicy)
}
if len(pod.Spec.Volumes) != 1 || pod.Spec.Volumes[0].PersistentVolumeClaim == nil {
t.Fatalf("expected one PVC volume, got %+v", pod.Spec.Volumes)
}
if pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != "demo-data" {
t.Fatalf("expected PVC claim name demo-data, got %s", pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName)
}
if len(pod.Spec.Containers) != 1 {
t.Fatalf("expected one container, got %d", len(pod.Spec.Containers))
}
c := pod.Spec.Containers[0]
if c.Image != "alpine:3" {
t.Fatalf("expected image alpine:3, got %s", c.Image)
}
if !c.Stdin || !c.TTY {
t.Fatalf("expected stdin and tty true on the container, got stdin=%v tty=%v", c.Stdin, c.TTY)
}
if len(c.VolumeMounts) != 1 || c.VolumeMounts[0].MountPath != "/data" {
t.Fatalf("expected one volume mount at /data, got %+v", c.VolumeMounts)
}
}
func TestBuildStoragePodSpecChownToSecurityContext(t *testing.T) {
entry := &storage.Entry{
Name: "demo-data",
Scheduler: storage.SchedulerK3s,
Chown: "herokuish",
}
pod, err := buildStoragePodSpec("p", "default", entry, StorageExecInput{Image: "alpine:3"})
if err != nil {
t.Fatalf("err: %v", err)
}
if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.RunAsUser == nil {
t.Fatalf("expected SecurityContext.RunAsUser to be populated for chown=herokuish")
}
// Don't assert exact UID because ResolveChownID consults docker for
// userns offset; only assert non-nil and non-zero.
if *pod.Spec.SecurityContext.RunAsUser == 0 {
t.Fatalf("expected non-zero UID for chown=herokuish, got 0")
}
}
func TestBuildStoragePodSpecAsUserOverride(t *testing.T) {
entry := &storage.Entry{
Name: "demo-data",
Scheduler: storage.SchedulerK3s,
Chown: "herokuish",
}
pod, err := buildStoragePodSpec("p", "default", entry, StorageExecInput{
Image: "alpine:3",
AsUser: "0",
})
if err != nil {
t.Fatalf("err: %v", err)
}
if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.RunAsUser == nil {
t.Fatalf("expected RunAsUser to be set")
}
if *pod.Spec.SecurityContext.RunAsUser != 0 {
t.Fatalf("expected RunAsUser to be 0 (override), got %d", *pod.Spec.SecurityContext.RunAsUser)
}
}
func TestBuildStoragePodSpecChownFalseLeavesDefaultUser(t *testing.T) {
entry := &storage.Entry{
Name: "demo-data",
Scheduler: storage.SchedulerK3s,
Chown: "false",
}
pod, err := buildStoragePodSpec("p", "default", entry, StorageExecInput{Image: "alpine:3"})
if err != nil {
t.Fatalf("err: %v", err)
}
if pod.Spec.SecurityContext != nil {
t.Fatalf("expected no SecurityContext when chown=false, got %+v", pod.Spec.SecurityContext)
}
}