refactor: fix issues with pod entering

- ensure we always wait for the specific pod we want to be ready
- enter the default container if none is specified
This commit is contained in:
Jose Diaz-Gonzalez
2024-01-19 19:26:06 -05:00
parent 5c4e1dfc39
commit 9cd0644883
2 changed files with 80 additions and 44 deletions

View File

@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"strings"
"time" "time"
"github.com/dokku/dokku/plugins/common" "github.com/dokku/dokku/plugins/common"
@@ -20,12 +21,12 @@ import (
) )
type EnterPodInput struct { type EnterPodInput struct {
AppName string Clientset KubernetesClient
Clientset KubernetesClient Command []string
Command []string Entrypoint string
Entrypoint string SelectedContainerName string
ProcessType string SelectedPod v1.Pod
SelectedPod v1.Pod WaitTimeout int
} }
// StartCommandInput contains all the information needed to get the start command // StartCommandInput contains all the information needed to get the start command
@@ -49,18 +50,20 @@ type StartCommandOutput struct {
} }
type WaitForPodBySelectorRunningInput struct { type WaitForPodBySelectorRunningInput struct {
Clientset KubernetesClient Clientset KubernetesClient
Namespace string Namespace string
Selector string LabelSelector string
Timeout int PodName string
Waiter func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc Timeout int
Waiter func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
} }
type WaitForPodToExistInput struct { type WaitForPodToExistInput struct {
Clientset KubernetesClient Clientset KubernetesClient
Namespace string Namespace string
RetryCount int RetryCount int
Selector string PodName string
LabelSelector string
} }
func createKubernetesNamespace(ctx context.Context, namespaceName string) error { func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
@@ -107,13 +110,42 @@ func enterPod(ctx context.Context, input EnterPodInput) error {
return fmt.Errorf("Error creating corev1 client: %w", err) return fmt.Errorf("Error creating corev1 client: %w", err)
} }
labelSelector := []string{}
for k, v := range input.SelectedPod.Labels {
labelSelector = append(labelSelector, fmt.Sprintf("%s=%s", k, v))
}
if input.WaitTimeout > 0 {
input.WaitTimeout = 5
}
err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
Clientset: input.Clientset,
Namespace: input.SelectedPod.Namespace,
LabelSelector: strings.Join(labelSelector, ","),
PodName: input.SelectedPod.Name,
Timeout: input.WaitTimeout,
Waiter: isPodReady,
})
if err != nil {
return fmt.Errorf("Error waiting for pod to be ready: %w", err)
}
defaultContainerName, hasDefaultContainer := input.SelectedPod.Annotations["kubectl.kubernetes.io/default-container"]
if input.SelectedContainerName == "" && hasDefaultContainer {
input.SelectedContainerName = defaultContainerName
}
if input.SelectedContainerName == "" {
return fmt.Errorf("No container specified and no default container found")
}
req := coreclient.RESTClient().Post(). req := coreclient.RESTClient().Post().
Resource("pods"). Resource("pods").
Namespace(input.SelectedPod.Namespace). Namespace(input.SelectedPod.Namespace).
Name(input.SelectedPod.Name). Name(input.SelectedPod.Name).
SubResource("exec") SubResource("exec")
req.Param("container", fmt.Sprintf("%s-%s", input.AppName, input.ProcessType)) req.Param("container", input.SelectedContainerName)
req.Param("stdin", "true") req.Param("stdin", "true")
req.Param("stdout", "true") req.Param("stdout", "true")
req.Param("stderr", "true") req.Param("stderr", "true")
@@ -270,21 +302,26 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp
func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error { func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{ pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
Clientset: input.Clientset, Clientset: input.Clientset,
Namespace: input.Namespace, LabelSelector: input.LabelSelector,
RetryCount: 3, Namespace: input.Namespace,
Selector: input.Selector, PodName: input.PodName,
RetryCount: 3,
}) })
if err != nil { if err != nil {
return fmt.Errorf("Error waiting for pod to exist: %w", err) return fmt.Errorf("Error waiting for pod to exist: %w", err)
} }
if len(pods) == 0 { if len(pods) == 0 {
return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.Selector) return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.LabelSelector)
} }
timeout := time.Duration(input.Timeout) * time.Second timeout := time.Duration(input.Timeout) * time.Second
for _, pod := range pods { for _, pod := range pods {
if input.PodName != "" && pod.Name != input.PodName {
break
}
if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil { if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
print("\n") print("\n")
return fmt.Errorf("Error waiting for pod to be ready: %w", err) return fmt.Errorf("Error waiting for pod to be ready: %w", err)
@@ -300,12 +337,21 @@ func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) ([]v1.
for i := 0; i < input.RetryCount; i++ { for i := 0; i < input.RetryCount; i++ {
pods, err = input.Clientset.ListPods(ctx, ListPodsInput{ pods, err = input.Clientset.ListPods(ctx, ListPodsInput{
Namespace: input.Namespace, Namespace: input.Namespace,
LabelSelector: input.Selector, LabelSelector: input.LabelSelector,
}) })
if err == nil { if err != nil {
time.Sleep(1 * time.Second)
}
if input.PodName == "" {
break break
} }
time.Sleep(1 * time.Second)
for _, pod := range pods {
if pod.Name == input.PodName {
break
}
}
} }
if err != nil { if err != nil {
return pods, fmt.Errorf("Error listing pods: %w", err) return pods, fmt.Errorf("Error listing pods: %w", err)

View File

@@ -181,7 +181,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
} }
defer os.RemoveAll(chartDir) defer os.RemoveAll(chartDir)
common.LogDebug(fmt.Sprintf("Using chart directory: %s", chartDir))
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil { if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
return fmt.Errorf("Error creating chart templates directory: %w", err) return fmt.Errorf("Error creating chart templates directory: %w", err)
} }
@@ -470,7 +469,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
return nil return nil
} }
common.LogDebug(fmt.Sprintf("%s %s %s %s", scheduler, appName, processType, podName))
clientset, err := NewKubernetesClient() clientset, err := NewKubernetesClient()
if err != nil { if err != nil {
return fmt.Errorf("Error creating kubernetes client: %w", err) return fmt.Errorf("Error creating kubernetes client: %w", err)
@@ -532,11 +530,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
selectedPod = pods[processIndex] selectedPod = pods[processIndex]
} }
processType, ok := selectedPod.Labels["dokku.com/process-type"]
if !ok {
return fmt.Errorf("Pod %s does not have a process type label", selectedPod.Name)
}
command := args command := args
if len(args) == 0 { if len(args) == 0 {
command = []string{"/bin/bash"} command = []string{"/bin/bash"}
@@ -554,12 +547,11 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
} }
return enterPod(ctx, EnterPodInput{ return enterPod(ctx, EnterPodInput{
AppName: appName,
Clientset: clientset, Clientset: clientset,
Command: command, Command: command,
Entrypoint: entrypoint, Entrypoint: entrypoint,
ProcessType: processType,
SelectedPod: selectedPod, SelectedPod: selectedPod,
WaitTimeout: 10,
}) })
} }
@@ -889,10 +881,10 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name) batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{ pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
Clientset: clientset, Clientset: clientset,
Namespace: namespace, Namespace: namespace,
RetryCount: 3, RetryCount: 3,
Selector: batchJobSelector, LabelSelector: batchJobSelector,
}) })
if err != nil { if err != nil {
return fmt.Errorf("Error waiting for pod to exist: %w", err) return fmt.Errorf("Error waiting for pod to exist: %w", err)
@@ -907,11 +899,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
} }
err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{ err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
Clientset: clientset, Clientset: clientset,
Namespace: namespace, Namespace: namespace,
Selector: batchJobSelector, LabelSelector: batchJobSelector,
Timeout: 300, Timeout: 300,
Waiter: isPodReady, Waiter: isPodReady,
}) })
if err != nil { if err != nil {
if errors.Is(err, conditions.ErrPodCompleted) { if errors.Is(err, conditions.ErrPodCompleted) {
@@ -970,11 +962,9 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
} }
case v1.PodRunning: case v1.PodRunning:
return enterPod(ctx, EnterPodInput{ return enterPod(ctx, EnterPodInput{
AppName: appName,
Clientset: clientset, Clientset: clientset,
Command: command, Command: command,
Entrypoint: entrypoint, Entrypoint: entrypoint,
ProcessType: processType,
SelectedPod: selectedPod, SelectedPod: selectedPod,
}) })
default: default: