fix: retire cron containers past their active deadline

Cron containers were never reaped once they exceeded their active deadline, so a hung cron task ran indefinitely instead of being retired after 24 hours as documented. `cron:run` now also accepts a `--ttl-seconds` argument, matching the one `dokku run` already takes.
This commit is contained in:
Jose Diaz-Gonzalez committed 2026-08-12 13:11:34 -04:00
1 parent 0bfd35dada
commit 96d726c43c
18 files changed
+1170 -906

No files matched your search

+7 -6
View File
@@ -577,12 +577,13 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
Args: words,
Annotations: annotations,
Cron: ProcessCron{
ID: cronTask.ID,
Hash: cronIDLabelValue(cronTask.ID),
Schedule: cronTask.Schedule,
Suffix: suffix,
Suspend: cronTask.Maintenance,
ConcurrencyPolicy: ProcessCronConcurrencyPolicy(concurrencyPolicy),
ID: cronTask.ID,
Hash: cronIDLabelValue(cronTask.ID),
Schedule: cronTask.Schedule,
Suffix: suffix,
Suspend: cronTask.Maintenance,
ConcurrencyPolicy: ProcessCronConcurrencyPolicy(concurrencyPolicy),
ActiveDeadlineSeconds: cron.DefaultTTLSeconds,
},
Labels: labels,
ProcessType: ProcessType_Cron,
@@ -3,6 +3,7 @@ package scheduler_k3s
import (
"errors"
"io"
"maps"
"os"
"path/filepath"
"strings"
@@ -148,6 +149,61 @@ func TestCronJobTemplateRendersLongCronID(t *testing.T) {
})
}
// TestCronJobTemplateRendersActiveDeadlineSeconds asserts that the job's
// activeDeadlineSeconds comes from the chart values rather than a literal
// baked into the template, so the deadline cannot drift from the value the
// docker-local scheduler stamps onto its cron containers. A release rendered
// from values that predate the field must still fall back to 24 hours.
func TestCronJobTemplateRendersActiveDeadlineSeconds(t *testing.T) {
baseValues := map[string]interface{}{
"id": "abcde",
"hash": "0123456789abcdef0123456789abcdef01234567",
"schedule": "5 5 5 5 5",
"suffix": "abcde",
"suspend": false,
"concurrency_policy": "Allow",
}
t.Run("from values", func(t *testing.T) {
values := maps.Clone(baseValues)
values["active_deadline_seconds"] = 600
if got := renderedActiveDeadlineSeconds(t, renderCronJobTemplate(t, values)); got != 600 {
t.Errorf("activeDeadlineSeconds = %d, want 600", got)
}
})
t.Run("defaults when absent", func(t *testing.T) {
if got := renderedActiveDeadlineSeconds(t, renderCronJobTemplate(t, maps.Clone(baseValues))); got != 86400 {
t.Errorf("activeDeadlineSeconds = %d, want 86400", got)
}
})
}
// renderedActiveDeadlineSeconds pulls jobTemplate.spec.activeDeadlineSeconds
// out of a rendered cron-job manifest.
func renderedActiveDeadlineSeconds(t *testing.T, manifest string) int {
t.Helper()
var doc map[string]interface{}
if err := yaml.NewDecoder(strings.NewReader(manifest)).Decode(&doc); err != nil {
t.Fatalf("yaml decode failed (would also fail in Kubernetes API): %v\nrendered:\n%s", err, manifest)
}
spec, _ := doc["spec"].(map[string]interface{})
jobTemplate, _ := spec["jobTemplate"].(map[string]interface{})
jobSpec, ok := jobTemplate["spec"].(map[string]interface{})
if !ok {
t.Fatalf("jobTemplate.spec missing from rendered manifest:\n%s", manifest)
}
deadline, ok := jobSpec["activeDeadlineSeconds"].(int)
if !ok {
t.Fatalf("activeDeadlineSeconds = %v (type %T); want an integer", jobSpec["activeDeadlineSeconds"], jobSpec["activeDeadlineSeconds"])
}
return deadline
}
// renderCronJobTemplate renders templates/chart/cron-job.yaml with the
// supplied cron-process values and returns the YAML manifest as a string.
// The chart is materialised in a tempdir alongside the shared _helpers.tpl
+7 -6
View File
@@ -306,12 +306,13 @@ const (
)
type ProcessCron struct {
ID string `yaml:"id"`
Hash string `yaml:"hash"`
Schedule string `yaml:"schedule"`
Suffix string `yaml:"suffix"`
Suspend bool `yaml:"suspend"`
ConcurrencyPolicy ProcessCronConcurrencyPolicy `yaml:"concurrency_policy"`
ID string `yaml:"id"`
Hash string `yaml:"hash"`
Schedule string `yaml:"schedule"`
Suffix string `yaml:"suffix"`
Suspend bool `yaml:"suspend"`
ConcurrencyPolicy ProcessCronConcurrencyPolicy `yaml:"concurrency_policy"`
ActiveDeadlineSeconds int64 `yaml:"active_deadline_seconds"`
}
type ProcessCronConcurrencyPolicy string
@@ -52,7 +52,7 @@ spec:
backoffLimit: 0
podReplacementPolicy: Failed
ttlSecondsAfterFinished: 60
activeDeadlineSeconds: 86400
activeDeadlineSeconds: {{ $config.cron.active_deadline_seconds | default 86400 }}
template:
metadata:
annotations: