feat: wait until helm charts are installed before marking a deploy as failed or completed

This ensures applications are actually up vs giving a false sense of security to users about the state of their applications.
This commit is contained in:
Jose Diaz-Gonzalez committed 2024-03-12 08:39:22 -04:00
1 parent 7c61b53cc8
commit c840fc5f9e
5 files changed
+13 -24

No files matched your search

+9
View File
@@ -226,6 +226,7 @@ func applyKedaClusterTriggerAuthentications(ctx context.Context, triggerType str
ReleaseName: fmt.Sprintf("keda-cluster-trigger-authentications-%s", triggerType), ReleaseName: fmt.Sprintf("keda-cluster-trigger-authentications-%s", triggerType),
RollbackOnFailure: true, RollbackOnFailure: true,
Timeout: timeoutDuration, Timeout: timeoutDuration,
Wait: true,
}) })
if err != nil { if err != nil {
return fmt.Errorf("Error installing keda-cluster-trigger-authentications-%s chart: %w", triggerType, err) return fmt.Errorf("Error installing keda-cluster-trigger-authentications-%s chart: %w", triggerType, err)
@@ -333,6 +334,7 @@ func applyClusterIssuers(ctx context.Context) error {
ReleaseName: "cluster-issuers", ReleaseName: "cluster-issuers",
RollbackOnFailure: true, RollbackOnFailure: true,
Timeout: timeoutDuration, Timeout: timeoutDuration,
Wait: true,
}) })
if err != nil { if err != nil {
return fmt.Errorf("Error installing cluster-issuer chart: %w", err) return fmt.Errorf("Error installing cluster-issuer chart: %w", err)
@@ -1398,6 +1400,11 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
return fmt.Errorf("Error creating helm agent: %w", err) return fmt.Errorf("Error creating helm agent: %w", err)
} }
timeoutDuration, err := time.ParseDuration("300s")
if err != nil {
return fmt.Errorf("Error parsing deploy timeout duration: %w", err)
}
err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{ err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
ChartPath: chart.ChartPath, ChartPath: chart.ChartPath,
Namespace: chart.Namespace, Namespace: chart.Namespace,
@@ -1405,6 +1412,8 @@ func installHelmCharts(ctx context.Context, clientset KubernetesClient, shouldIn
RepoURL: chart.RepoURL, RepoURL: chart.RepoURL,
Values: values, Values: values,
Version: chart.Version, Version: chart.Version,
Timeout: timeoutDuration,
Wait: true,
}) })
if err != nil { if err != nil {
return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err) return fmt.Errorf("Error installing chart %s: %w", chart.ChartPath, err)
+3 -2
View File
@@ -51,6 +51,7 @@ type ChartInput struct {
RepoURL string RepoURL string
RollbackOnFailure bool RollbackOnFailure bool
Timeout time.Duration Timeout time.Duration
Wait bool
Version string Version string
Values map[string]interface{} Values map[string]interface{}
} }
@@ -239,7 +240,7 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error {
client.Namespace = namespace client.Namespace = namespace
client.ReleaseName = input.ReleaseName client.ReleaseName = input.ReleaseName
client.Timeout = input.Timeout client.Timeout = input.Timeout
client.Wait = false client.Wait = input.Wait
settings := cli.New() settings := cli.New()
if input.RepoURL != "" { if input.RepoURL != "" {
@@ -314,7 +315,7 @@ func (h *HelmAgent) UpgradeChart(ctx context.Context, input ChartInput) error {
client.MaxHistory = 10 client.MaxHistory = 10
client.Namespace = namespace client.Namespace = namespace
client.Timeout = input.Timeout client.Timeout = input.Timeout
client.Wait = false client.Wait = input.Wait
if input.RepoURL != "" { if input.RepoURL != "" {
client.RepoURL = input.RepoURL client.RepoURL = input.RepoURL
} }
+1
View File
@@ -594,6 +594,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
ReleaseName: appName, ReleaseName: appName,
RollbackOnFailure: allowRollbacks, RollbackOnFailure: allowRollbacks,
Timeout: timeoutDuration, Timeout: timeoutDuration,
Wait: true,
}) })
if err != nil { if err != nil {
return err return err
-20
View File
@@ -44,11 +44,6 @@ teardown_() {
echo "status: $status" echo "status: $status"
assert_success assert_success
run /bin/bash -c "sleep 20"
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
# include autoscaling tests # include autoscaling tests
@@ -128,11 +123,6 @@ teardown_() {
echo "status: $status" echo "status: $status"
assert_success assert_success
run /bin/bash -c "sleep 20"
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'" run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'"
@@ -182,11 +172,6 @@ teardown_() {
echo "status: $status" echo "status: $status"
assert_success assert_success
run /bin/bash -c "sleep 20"
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
} }
@@ -237,11 +222,6 @@ teardown_() {
echo "status: $status" echo "status: $status"
assert_success assert_success
run /bin/bash -c "sleep 20"
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.metadata.annotations.\"test.dokku.com/resource-type\"'" run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.metadata.annotations.\"test.dokku.com/resource-type\"'"
-2
View File
@@ -668,8 +668,6 @@ install_k3s() {
echo "output: $output" echo "output: $output"
echo "status: $status" echo "status: $status"
assert_success assert_success
sleep 20
} }
uninstall_k3s() { uninstall_k3s() {