refactor: fix issues with pod entering
- ensure we always wait for the specific pod we want to be ready - enter the default container if none is specified
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dokku/dokku/plugins/common"
|
"github.com/dokku/dokku/plugins/common"
|
||||||
@@ -20,12 +21,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type EnterPodInput struct {
|
type EnterPodInput struct {
|
||||||
AppName string
|
Clientset KubernetesClient
|
||||||
Clientset KubernetesClient
|
Command []string
|
||||||
Command []string
|
Entrypoint string
|
||||||
Entrypoint string
|
SelectedContainerName string
|
||||||
ProcessType string
|
SelectedPod v1.Pod
|
||||||
SelectedPod v1.Pod
|
WaitTimeout int
|
||||||
}
|
}
|
||||||
|
|
||||||
// StartCommandInput contains all the information needed to get the start command
|
// StartCommandInput contains all the information needed to get the start command
|
||||||
@@ -49,18 +50,20 @@ type StartCommandOutput struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type WaitForPodBySelectorRunningInput struct {
|
type WaitForPodBySelectorRunningInput struct {
|
||||||
Clientset KubernetesClient
|
Clientset KubernetesClient
|
||||||
Namespace string
|
Namespace string
|
||||||
Selector string
|
LabelSelector string
|
||||||
Timeout int
|
PodName string
|
||||||
Waiter func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
|
Timeout int
|
||||||
|
Waiter func(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc
|
||||||
}
|
}
|
||||||
|
|
||||||
type WaitForPodToExistInput struct {
|
type WaitForPodToExistInput struct {
|
||||||
Clientset KubernetesClient
|
Clientset KubernetesClient
|
||||||
Namespace string
|
Namespace string
|
||||||
RetryCount int
|
RetryCount int
|
||||||
Selector string
|
PodName string
|
||||||
|
LabelSelector string
|
||||||
}
|
}
|
||||||
|
|
||||||
func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
|
func createKubernetesNamespace(ctx context.Context, namespaceName string) error {
|
||||||
@@ -107,13 +110,42 @@ func enterPod(ctx context.Context, input EnterPodInput) error {
|
|||||||
return fmt.Errorf("Error creating corev1 client: %w", err)
|
return fmt.Errorf("Error creating corev1 client: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
labelSelector := []string{}
|
||||||
|
for k, v := range input.SelectedPod.Labels {
|
||||||
|
labelSelector = append(labelSelector, fmt.Sprintf("%s=%s", k, v))
|
||||||
|
}
|
||||||
|
|
||||||
|
if input.WaitTimeout > 0 {
|
||||||
|
input.WaitTimeout = 5
|
||||||
|
}
|
||||||
|
|
||||||
|
err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
|
||||||
|
Clientset: input.Clientset,
|
||||||
|
Namespace: input.SelectedPod.Namespace,
|
||||||
|
LabelSelector: strings.Join(labelSelector, ","),
|
||||||
|
PodName: input.SelectedPod.Name,
|
||||||
|
Timeout: input.WaitTimeout,
|
||||||
|
Waiter: isPodReady,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error waiting for pod to be ready: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defaultContainerName, hasDefaultContainer := input.SelectedPod.Annotations["kubectl.kubernetes.io/default-container"]
|
||||||
|
if input.SelectedContainerName == "" && hasDefaultContainer {
|
||||||
|
input.SelectedContainerName = defaultContainerName
|
||||||
|
}
|
||||||
|
if input.SelectedContainerName == "" {
|
||||||
|
return fmt.Errorf("No container specified and no default container found")
|
||||||
|
}
|
||||||
|
|
||||||
req := coreclient.RESTClient().Post().
|
req := coreclient.RESTClient().Post().
|
||||||
Resource("pods").
|
Resource("pods").
|
||||||
Namespace(input.SelectedPod.Namespace).
|
Namespace(input.SelectedPod.Namespace).
|
||||||
Name(input.SelectedPod.Name).
|
Name(input.SelectedPod.Name).
|
||||||
SubResource("exec")
|
SubResource("exec")
|
||||||
|
|
||||||
req.Param("container", fmt.Sprintf("%s-%s", input.AppName, input.ProcessType))
|
req.Param("container", input.SelectedContainerName)
|
||||||
req.Param("stdin", "true")
|
req.Param("stdin", "true")
|
||||||
req.Param("stdout", "true")
|
req.Param("stdout", "true")
|
||||||
req.Param("stderr", "true")
|
req.Param("stderr", "true")
|
||||||
@@ -270,21 +302,26 @@ func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namesp
|
|||||||
|
|
||||||
func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
|
func waitForPodBySelectorRunning(ctx context.Context, input WaitForPodBySelectorRunningInput) error {
|
||||||
pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
|
pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
|
||||||
Clientset: input.Clientset,
|
Clientset: input.Clientset,
|
||||||
Namespace: input.Namespace,
|
LabelSelector: input.LabelSelector,
|
||||||
RetryCount: 3,
|
Namespace: input.Namespace,
|
||||||
Selector: input.Selector,
|
PodName: input.PodName,
|
||||||
|
RetryCount: 3,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error waiting for pod to exist: %w", err)
|
return fmt.Errorf("Error waiting for pod to exist: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(pods) == 0 {
|
if len(pods) == 0 {
|
||||||
return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.Selector)
|
return fmt.Errorf("no pods in %s with selector %s", input.Namespace, input.LabelSelector)
|
||||||
}
|
}
|
||||||
|
|
||||||
timeout := time.Duration(input.Timeout) * time.Second
|
timeout := time.Duration(input.Timeout) * time.Second
|
||||||
for _, pod := range pods {
|
for _, pod := range pods {
|
||||||
|
if input.PodName != "" && pod.Name != input.PodName {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
|
if err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, input.Waiter(ctx, input.Clientset, pod.Name, pod.Namespace)); err != nil {
|
||||||
print("\n")
|
print("\n")
|
||||||
return fmt.Errorf("Error waiting for pod to be ready: %w", err)
|
return fmt.Errorf("Error waiting for pod to be ready: %w", err)
|
||||||
@@ -300,12 +337,21 @@ func waitForPodToExist(ctx context.Context, input WaitForPodToExistInput) ([]v1.
|
|||||||
for i := 0; i < input.RetryCount; i++ {
|
for i := 0; i < input.RetryCount; i++ {
|
||||||
pods, err = input.Clientset.ListPods(ctx, ListPodsInput{
|
pods, err = input.Clientset.ListPods(ctx, ListPodsInput{
|
||||||
Namespace: input.Namespace,
|
Namespace: input.Namespace,
|
||||||
LabelSelector: input.Selector,
|
LabelSelector: input.LabelSelector,
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
|
time.Sleep(1 * time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
if input.PodName == "" {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
time.Sleep(1 * time.Second)
|
|
||||||
|
for _, pod := range pods {
|
||||||
|
if pod.Name == input.PodName {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return pods, fmt.Errorf("Error listing pods: %w", err)
|
return pods, fmt.Errorf("Error listing pods: %w", err)
|
||||||
|
|||||||
@@ -181,7 +181,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(chartDir)
|
defer os.RemoveAll(chartDir)
|
||||||
|
|
||||||
common.LogDebug(fmt.Sprintf("Using chart directory: %s", chartDir))
|
|
||||||
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
|
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
|
||||||
return fmt.Errorf("Error creating chart templates directory: %w", err)
|
return fmt.Errorf("Error creating chart templates directory: %w", err)
|
||||||
}
|
}
|
||||||
@@ -470,7 +469,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
common.LogDebug(fmt.Sprintf("%s %s %s %s", scheduler, appName, processType, podName))
|
|
||||||
clientset, err := NewKubernetesClient()
|
clientset, err := NewKubernetesClient()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
||||||
@@ -532,11 +530,6 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
|
|||||||
selectedPod = pods[processIndex]
|
selectedPod = pods[processIndex]
|
||||||
}
|
}
|
||||||
|
|
||||||
processType, ok := selectedPod.Labels["dokku.com/process-type"]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("Pod %s does not have a process type label", selectedPod.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
command := args
|
command := args
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
command = []string{"/bin/bash"}
|
command = []string{"/bin/bash"}
|
||||||
@@ -554,12 +547,11 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
return enterPod(ctx, EnterPodInput{
|
return enterPod(ctx, EnterPodInput{
|
||||||
AppName: appName,
|
|
||||||
Clientset: clientset,
|
Clientset: clientset,
|
||||||
Command: command,
|
Command: command,
|
||||||
Entrypoint: entrypoint,
|
Entrypoint: entrypoint,
|
||||||
ProcessType: processType,
|
|
||||||
SelectedPod: selectedPod,
|
SelectedPod: selectedPod,
|
||||||
|
WaitTimeout: 10,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -889,10 +881,10 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
|
|||||||
|
|
||||||
batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
|
batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
|
||||||
pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
|
pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
|
||||||
Clientset: clientset,
|
Clientset: clientset,
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
RetryCount: 3,
|
RetryCount: 3,
|
||||||
Selector: batchJobSelector,
|
LabelSelector: batchJobSelector,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error waiting for pod to exist: %w", err)
|
return fmt.Errorf("Error waiting for pod to exist: %w", err)
|
||||||
@@ -907,11 +899,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
|
|||||||
}
|
}
|
||||||
|
|
||||||
err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
|
err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
|
||||||
Clientset: clientset,
|
Clientset: clientset,
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Selector: batchJobSelector,
|
LabelSelector: batchJobSelector,
|
||||||
Timeout: 300,
|
Timeout: 300,
|
||||||
Waiter: isPodReady,
|
Waiter: isPodReady,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, conditions.ErrPodCompleted) {
|
if errors.Is(err, conditions.ErrPodCompleted) {
|
||||||
@@ -970,11 +962,9 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
|
|||||||
}
|
}
|
||||||
case v1.PodRunning:
|
case v1.PodRunning:
|
||||||
return enterPod(ctx, EnterPodInput{
|
return enterPod(ctx, EnterPodInput{
|
||||||
AppName: appName,
|
|
||||||
Clientset: clientset,
|
Clientset: clientset,
|
||||||
Command: command,
|
Command: command,
|
||||||
Entrypoint: entrypoint,
|
Entrypoint: entrypoint,
|
||||||
ProcessType: processType,
|
|
||||||
SelectedPod: selectedPod,
|
SelectedPod: selectedPod,
|
||||||
})
|
})
|
||||||
default:
|
default:
|
||||||
|
|||||||
Reference in New Issue
Block a user