feat: delegate storage:exec to scheduler plugins, harden the flow
Moves the actual exec out of the storage plugin and into a new scheduler-storage-exec plugn trigger. scheduler-docker-local does docker run with TTY-aware -it/-i selection and --user derived from entry.Chown; scheduler-k3s creates a throwaway Pod via the kubernetes API, waits for it to reach Running with structured error reporting (ImagePullBackOff and friends are surfaced from the container status verbatim, no kubectl involvement), execs the user command via the existing SPDY plumbing in k8s.go, and deletes the Pod on the way out. (Entry).Validate now accepts either an absolute path or a docker named-volume token for docker-local entries so the migration synthesizer's named-volume legacy entries work cleanly. storage:exec gains --as-user for one-off uid overrides, propagates the underlying tool's exit code via os.Exit, and detects TTY/interactive mode from os.Stdin so non-interactive scripted use no longer trips over docker's input-device-is-not-a-tty error.
This commit is contained in:
17 files changed
+790
-72
No files matched your search
@@ -1,5 +1,5 @@
|
||||
SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
|
||||
TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/storage-create triggers/storage-destroy triggers/storage-status
|
||||
TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec
|
||||
BUILD = commands subcommands triggers
|
||||
PLUGIN_NAME = scheduler-k3s
|
||||
|
||||
|
||||
@@ -19,6 +19,9 @@ func main() {
|
||||
parts := strings.Split(os.Args[0], "/")
|
||||
trigger := parts[len(parts)-1]
|
||||
podIdentifier := flag.String("container-id", "", "--container-id: A pod identifier")
|
||||
storageExecInteractive := flag.Bool("interactive", false, "--interactive: stdin is open (storage:exec)")
|
||||
storageExecTty := flag.Bool("tty", false, "--tty: stdin is a terminal (storage:exec)")
|
||||
storageExecAsUser := flag.String("as-user", "", "--as-user: numeric uid override (storage:exec)")
|
||||
flag.Parse()
|
||||
|
||||
var err error
|
||||
@@ -168,6 +171,27 @@ func main() {
|
||||
case "storage-status":
|
||||
entryName := flag.Arg(0)
|
||||
err = scheduler_k3s.TriggerStorageStatus(context.Background(), entryName)
|
||||
case "scheduler-storage-exec":
|
||||
positional := flag.Args()
|
||||
if len(positional) < 3 {
|
||||
err = fmt.Errorf("scheduler-storage-exec requires <scheduler> <entry-name> <image>")
|
||||
break
|
||||
}
|
||||
schedulerName := positional[0]
|
||||
entryName := positional[1]
|
||||
image := positional[2]
|
||||
var cmd []string
|
||||
if len(positional) > 3 {
|
||||
cmd = positional[3:]
|
||||
}
|
||||
err = scheduler_k3s.TriggerSchedulerStorageExec(context.Background(), schedulerName, scheduler_k3s.StorageExecInput{
|
||||
EntryName: entryName,
|
||||
Image: image,
|
||||
Interactive: *storageExecInteractive,
|
||||
Tty: *storageExecTty,
|
||||
AsUser: *storageExecAsUser,
|
||||
Command: cmd,
|
||||
})
|
||||
default:
|
||||
err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
package scheduler_k3s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/dokku/dokku/plugins/common"
|
||||
"github.com/dokku/dokku/plugins/storage"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilexec "k8s.io/client-go/util/exec"
|
||||
)
|
||||
|
||||
// StorageExecInput captures the inputs forwarded by the storage plugin
|
||||
// over the scheduler-storage-exec trigger.
|
||||
type StorageExecInput struct {
|
||||
EntryName string
|
||||
Image string
|
||||
Interactive bool
|
||||
Tty bool
|
||||
AsUser string
|
||||
Command []string
|
||||
}
|
||||
|
||||
// TriggerSchedulerStorageExec runs an interactive or non-interactive
|
||||
// command against a k3s storage entry by creating a throwaway Pod that
|
||||
// mounts the PVC at /data, exec-ing the user's command via the
|
||||
// kubernetes API, and deleting the Pod afterwards. Errors at every
|
||||
// stage (PVC missing, image pull failure, etc.) come through the
|
||||
// structured Kubernetes API objects rather than kubectl-shaped strings.
|
||||
func TriggerSchedulerStorageExec(ctx context.Context, scheduler string, input StorageExecInput) error {
|
||||
if scheduler != storage.SchedulerK3s {
|
||||
return nil
|
||||
}
|
||||
if !storage.EntryExists(input.EntryName) {
|
||||
return fmt.Errorf("storage entry %q does not exist", input.EntryName)
|
||||
}
|
||||
entry, err := storage.LoadEntry(input.EntryName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.Scheduler != storage.SchedulerK3s {
|
||||
return fmt.Errorf("storage entry %q has scheduler %q, not k3s", entry.Name, entry.Scheduler)
|
||||
}
|
||||
|
||||
if err := isKubernetesAvailable(); err != nil {
|
||||
return fmt.Errorf("kubernetes not available: %w", err)
|
||||
}
|
||||
|
||||
clientset, err := NewKubernetesClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
namespace := entry.Namespace
|
||||
if namespace == "" {
|
||||
namespace = "default"
|
||||
}
|
||||
|
||||
if _, err := clientset.Client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, entry.Name, metav1.GetOptions{}); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("PVC %s not found in namespace %s", entry.Name, namespace)
|
||||
}
|
||||
return fmt.Errorf("error reading PVC %s in namespace %s: %w", entry.Name, namespace, err)
|
||||
}
|
||||
|
||||
podName := fmt.Sprintf("dokku-storage-exec-%s-%d", entry.Name, time.Now().UnixNano()/int64(time.Millisecond)%1000000)
|
||||
pod, err := buildStoragePodSpec(podName, namespace, entry, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
common.LogVerboseQuiet(fmt.Sprintf("Creating exec pod %s/%s", namespace, podName))
|
||||
if _, err := clientset.Client.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
|
||||
return fmt.Errorf("error creating exec pod %s/%s: %w", namespace, podName, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
bg := context.Background()
|
||||
gracePeriod := int64(0)
|
||||
_ = clientset.Client.CoreV1().Pods(namespace).Delete(bg, podName, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod})
|
||||
}()
|
||||
|
||||
if err := waitForPodRunning(ctx, clientset, namespace, podName, 60*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
command := input.Command
|
||||
if len(command) == 0 {
|
||||
command = []string{"sh", "-c", "command -v bash >/dev/null 2>&1 && exec bash || exec sh"}
|
||||
}
|
||||
|
||||
execErr := clientset.ExecCommand(ctx, ExecCommandInput{
|
||||
Name: podName,
|
||||
Namespace: namespace,
|
||||
ContainerName: "exec",
|
||||
Command: command,
|
||||
})
|
||||
if execErr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var coded utilexec.CodeExitError
|
||||
if errors.As(execErr, &coded) {
|
||||
os.Exit(coded.Code)
|
||||
}
|
||||
return execErr
|
||||
}
|
||||
|
||||
// buildStoragePodSpec assembles the throwaway Pod that mounts the PVC
|
||||
// for storage:exec. Split out so the unit test can verify the spec
|
||||
// without hitting a cluster.
|
||||
func buildStoragePodSpec(name string, namespace string, entry *storage.Entry, input StorageExecInput) (*corev1.Pod, error) {
|
||||
uid, err := resolveStorageExecUID(entry.Chown, input.AsUser)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: map[string]string{
|
||||
"app.kubernetes.io/managed-by": "dokku",
|
||||
"dokku.com/storage-entry": entry.Name,
|
||||
"dokku.com/purpose": "storage-exec",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
Volumes: []corev1.Volume{
|
||||
{
|
||||
Name: "data",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: entry.Name,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "exec",
|
||||
Image: input.Image,
|
||||
Command: []string{"sleep", "infinity"},
|
||||
Stdin: input.Interactive,
|
||||
TTY: input.Tty,
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "data", MountPath: "/data"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if uid != nil {
|
||||
pod.Spec.SecurityContext = &corev1.PodSecurityContext{
|
||||
RunAsUser: uid,
|
||||
RunAsGroup: uid,
|
||||
FSGroup: uid,
|
||||
}
|
||||
}
|
||||
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
// resolveStorageExecUID picks the UID to run the exec Pod as. Returns
|
||||
// nil to mean "leave the image's default user in place".
|
||||
func resolveStorageExecUID(chown string, asUser string) (*int64, error) {
|
||||
if asUser != "" {
|
||||
n, err := strconv.ParseInt(asUser, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("--as-user must be a numeric uid, got %q", asUser)
|
||||
}
|
||||
return &n, nil
|
||||
}
|
||||
if chown == "" || chown == "false" {
|
||||
return nil, nil
|
||||
}
|
||||
uidStr, err := storage.ResolveChownID(chown)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if uidStr == "" || uidStr == "false" {
|
||||
return nil, nil
|
||||
}
|
||||
n, err := strconv.ParseInt(uidStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse resolved chown UID %q: %w", uidStr, err)
|
||||
}
|
||||
return &n, nil
|
||||
}
|
||||
|
||||
// waitForPodRunning polls the Pod until its phase is Running, surfacing
|
||||
// the structured Kubernetes API state on failure (image pull failures,
|
||||
// scheduling problems, etc.) rather than a generic timeout.
|
||||
func waitForPodRunning(ctx context.Context, clientset KubernetesClient, namespace, name string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
pod, err := clientset.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading exec pod %s/%s: %w", namespace, name, err)
|
||||
}
|
||||
switch pod.Status.Phase {
|
||||
case corev1.PodRunning:
|
||||
return nil
|
||||
case corev1.PodFailed, corev1.PodSucceeded:
|
||||
return fmt.Errorf("exec pod %s/%s reached terminal phase %s before exec could attach", namespace, name, pod.Status.Phase)
|
||||
}
|
||||
|
||||
for _, cs := range pod.Status.ContainerStatuses {
|
||||
if cs.State.Waiting == nil {
|
||||
continue
|
||||
}
|
||||
reason := cs.State.Waiting.Reason
|
||||
switch reason {
|
||||
case "ImagePullBackOff", "ErrImagePull", "CreateContainerConfigError", "InvalidImageName":
|
||||
return fmt.Errorf("exec pod %s/%s container %q failed to start: %s: %s", namespace, name, cs.Name, reason, cs.State.Waiting.Message)
|
||||
}
|
||||
}
|
||||
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("exec pod %s/%s did not reach Running within %s (current phase: %s)", namespace, name, timeout, pod.Status.Phase)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package scheduler_k3s
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/dokku/dokku/plugins/storage"
|
||||
)
|
||||
|
||||
func TestBuildStoragePodSpec(t *testing.T) {
|
||||
entry := &storage.Entry{
|
||||
Name: "demo-data",
|
||||
Scheduler: storage.SchedulerK3s,
|
||||
Namespace: "dokku",
|
||||
}
|
||||
pod, err := buildStoragePodSpec("dokku-storage-exec-demo-data-1", "dokku", entry, StorageExecInput{
|
||||
Image: "alpine:3",
|
||||
Interactive: true,
|
||||
Tty: true,
|
||||
Command: []string{"sh"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("buildStoragePodSpec err: %v", err)
|
||||
}
|
||||
|
||||
if pod.Spec.RestartPolicy != "Never" {
|
||||
t.Fatalf("expected RestartPolicy=Never, got %s", pod.Spec.RestartPolicy)
|
||||
}
|
||||
if len(pod.Spec.Volumes) != 1 || pod.Spec.Volumes[0].PersistentVolumeClaim == nil {
|
||||
t.Fatalf("expected one PVC volume, got %+v", pod.Spec.Volumes)
|
||||
}
|
||||
if pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != "demo-data" {
|
||||
t.Fatalf("expected PVC claim name demo-data, got %s", pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName)
|
||||
}
|
||||
if len(pod.Spec.Containers) != 1 {
|
||||
t.Fatalf("expected one container, got %d", len(pod.Spec.Containers))
|
||||
}
|
||||
c := pod.Spec.Containers[0]
|
||||
if c.Image != "alpine:3" {
|
||||
t.Fatalf("expected image alpine:3, got %s", c.Image)
|
||||
}
|
||||
if !c.Stdin || !c.TTY {
|
||||
t.Fatalf("expected stdin and tty true on the container, got stdin=%v tty=%v", c.Stdin, c.TTY)
|
||||
}
|
||||
if len(c.VolumeMounts) != 1 || c.VolumeMounts[0].MountPath != "/data" {
|
||||
t.Fatalf("expected one volume mount at /data, got %+v", c.VolumeMounts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildStoragePodSpecChownToSecurityContext(t *testing.T) {
|
||||
entry := &storage.Entry{
|
||||
Name: "demo-data",
|
||||
Scheduler: storage.SchedulerK3s,
|
||||
Chown: "herokuish",
|
||||
}
|
||||
pod, err := buildStoragePodSpec("p", "default", entry, StorageExecInput{Image: "alpine:3"})
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.RunAsUser == nil {
|
||||
t.Fatalf("expected SecurityContext.RunAsUser to be populated for chown=herokuish")
|
||||
}
|
||||
// Don't assert exact UID because ResolveChownID consults docker for
|
||||
// userns offset; only assert non-nil and non-zero.
|
||||
if *pod.Spec.SecurityContext.RunAsUser == 0 {
|
||||
t.Fatalf("expected non-zero UID for chown=herokuish, got 0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildStoragePodSpecAsUserOverride(t *testing.T) {
|
||||
entry := &storage.Entry{
|
||||
Name: "demo-data",
|
||||
Scheduler: storage.SchedulerK3s,
|
||||
Chown: "herokuish",
|
||||
}
|
||||
pod, err := buildStoragePodSpec("p", "default", entry, StorageExecInput{
|
||||
Image: "alpine:3",
|
||||
AsUser: "0",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.RunAsUser == nil {
|
||||
t.Fatalf("expected RunAsUser to be set")
|
||||
}
|
||||
if *pod.Spec.SecurityContext.RunAsUser != 0 {
|
||||
t.Fatalf("expected RunAsUser to be 0 (override), got %d", *pod.Spec.SecurityContext.RunAsUser)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildStoragePodSpecChownFalseLeavesDefaultUser(t *testing.T) {
|
||||
entry := &storage.Entry{
|
||||
Name: "demo-data",
|
||||
Scheduler: storage.SchedulerK3s,
|
||||
Chown: "false",
|
||||
}
|
||||
pod, err := buildStoragePodSpec("p", "default", entry, StorageExecInput{Image: "alpine:3"})
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if pod.Spec.SecurityContext != nil {
|
||||
t.Fatalf("expected no SecurityContext when chown=false, got %+v", pod.Spec.SecurityContext)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user