Files
dokku/plugins/scheduler-k3s/triggers_test.go
T

343 lines
12 KiB
Go

package scheduler_k3s
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/dokku/dokku/plugins/common"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
)
func setupChartMigrationTest(t *testing.T) {
t.Helper()
RegisterTestingT(t)
t.Setenv("PLUGIN_PATH", "/var/lib/dokku/plugins")
t.Setenv("PLUGIN_ENABLED_PATH", "/var/lib/dokku/plugins/enabled")
t.Setenv("DOKKU_LIB_ROOT", t.TempDir())
t.Setenv("DOKKU_SYSTEM_USER", "root")
t.Setenv("DOKKU_SYSTEM_GROUP", "root")
Expect(common.PropertySetup("scheduler-k3s")).To(Succeed())
}
func setupAnnotationsLabelsMigrationTest(t *testing.T, apps ...string) {
t.Helper()
setupChartMigrationTest(t)
dokkuRoot := t.TempDir()
t.Setenv("DOKKU_ROOT", dokkuRoot)
for _, appName := range apps {
Expect(os.MkdirAll(filepath.Join(dokkuRoot, appName), 0755)).To(Succeed())
}
}
// writeLegacyAnnotationLabelFile seeds a property file directly using
// PropertyListWrite, matching the pre-migration on-disk layout (one
// "key: value" entry per line).
func writeLegacyAnnotationLabelFile(t *testing.T, scope string, property string, entries map[string]string) {
t.Helper()
lines := []string{}
for k, v := range entries {
lines = append(lines, k+": "+v)
}
Expect(common.PropertyListWrite("scheduler-k3s", scope, property, lines)).To(Succeed())
}
func TestMigrateChartPropertiesToMapFormat(t *testing.T) {
setupChartMigrationTest(t)
Expect(common.PropertyWrite("scheduler-k3s", "--global", "chart.cert-manager.installCRDs", "false")).To(Succeed())
Expect(common.PropertyWrite("scheduler-k3s", "--global", "chart.cert-manager.version", "1.13.3")).To(Succeed())
Expect(common.PropertyWrite("scheduler-k3s", "--global", "chart.traefik.replicas", "2")).To(Succeed())
Expect(migrateChartPropertiesToMapFormat()).To(Succeed())
certManager, err := common.PropertyMapGet("scheduler-k3s", "--global", "chart-overrides.cert-manager")
Expect(err).NotTo(HaveOccurred())
Expect(certManager).To(Equal(map[string]string{
"installCRDs": "false",
"version": "1.13.3",
}))
traefik, err := common.PropertyMapGet("scheduler-k3s", "--global", "chart-overrides.traefik")
Expect(err).NotTo(HaveOccurred())
Expect(traefik).To(Equal(map[string]string{"replicas": "2"}))
Expect(common.PropertyExists("scheduler-k3s", "--global", "chart.cert-manager.installCRDs")).To(BeFalse())
Expect(common.PropertyExists("scheduler-k3s", "--global", "chart.cert-manager.version")).To(BeFalse())
Expect(common.PropertyExists("scheduler-k3s", "--global", "chart.traefik.replicas")).To(BeFalse())
}
func TestMigrateChartPropertiesToMapFormatIsIdempotent(t *testing.T) {
setupChartMigrationTest(t)
Expect(common.PropertyWrite("scheduler-k3s", "--global", "chart.cert-manager.version", "1.13.3")).To(Succeed())
Expect(migrateChartPropertiesToMapFormat()).To(Succeed())
Expect(migrateChartPropertiesToMapFormat()).To(Succeed())
got, err := common.PropertyMapGet("scheduler-k3s", "--global", "chart-overrides.cert-manager")
Expect(err).NotTo(HaveOccurred())
Expect(got).To(Equal(map[string]string{"version": "1.13.3"}))
}
func TestMigrateChartPropertiesToMapFormatPreservesExistingNonOverlappingKeys(t *testing.T) {
setupChartMigrationTest(t)
Expect(common.PropertyMapWrite("scheduler-k3s", "--global", "chart-overrides.cert-manager", map[string]string{
"installCRDs": "true",
"service.annotations.prometheus.io/scrape": "true",
})).To(Succeed())
Expect(common.PropertyWrite("scheduler-k3s", "--global", "chart.cert-manager.installCRDs", "false")).To(Succeed())
Expect(common.PropertyWrite("scheduler-k3s", "--global", "chart.cert-manager.version", "1.13.3")).To(Succeed())
Expect(migrateChartPropertiesToMapFormat()).To(Succeed())
got, err := common.PropertyMapGet("scheduler-k3s", "--global", "chart-overrides.cert-manager")
Expect(err).NotTo(HaveOccurred())
Expect(got).To(Equal(map[string]string{
"installCRDs": "false",
"service.annotations.prometheus.io/scrape": "true",
"version": "1.13.3",
}))
}
func TestMigrateChartPropertiesToMapFormatNoLegacyIsNoOp(t *testing.T) {
setupChartMigrationTest(t)
Expect(migrateChartPropertiesToMapFormat()).To(Succeed())
for _, chart := range HelmCharts {
Expect(common.PropertyExists("scheduler-k3s", "--global", "chart-overrides."+chart.ReleaseName)).To(BeFalse())
}
}
func TestMigrateAnnotationsLabelsToMapFormat(t *testing.T) {
setupAnnotationsLabelsMigrationTest(t, "node-js-app")
writeLegacyAnnotationLabelFile(t, "node-js-app", "web.deployment", map[string]string{
"prometheus.io/scrape": "true",
"team": "platform",
})
writeLegacyAnnotationLabelFile(t, "node-js-app", "labels.global.service", map[string]string{
"app.kubernetes.io/managed-by": "dokku",
})
writeLegacyAnnotationLabelFile(t, "--global", "global.deployment", map[string]string{
"owner": "ops",
})
Expect(migrateAnnotationsLabelsToMapFormat()).To(Succeed())
appAnnotations, err := common.PropertyMapGet("scheduler-k3s", "node-js-app", "web.deployment")
Expect(err).NotTo(HaveOccurred())
Expect(appAnnotations).To(Equal(map[string]string{
"prometheus.io/scrape": "true",
"team": "platform",
}))
appLabels, err := common.PropertyMapGet("scheduler-k3s", "node-js-app", "labels.global.service")
Expect(err).NotTo(HaveOccurred())
Expect(appLabels).To(Equal(map[string]string{
"app.kubernetes.io/managed-by": "dokku",
}))
globalAnnotations, err := common.PropertyMapGet("scheduler-k3s", "--global", "global.deployment")
Expect(err).NotTo(HaveOccurred())
Expect(globalAnnotations).To(Equal(map[string]string{"owner": "ops"}))
raw, err := os.ReadFile(filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "config", "scheduler-k3s", "node-js-app", "web.deployment"))
Expect(err).NotTo(HaveOccurred())
var decoded map[string]string
Expect(json.Unmarshal(raw, &decoded)).To(Succeed())
Expect(decoded).To(Equal(appAnnotations))
}
func TestMigrateAnnotationsLabelsToMapFormatIsIdempotent(t *testing.T) {
setupAnnotationsLabelsMigrationTest(t, "node-js-app")
writeLegacyAnnotationLabelFile(t, "node-js-app", "web.deployment", map[string]string{
"prometheus.io/scrape": "true",
})
Expect(migrateAnnotationsLabelsToMapFormat()).To(Succeed())
Expect(migrateAnnotationsLabelsToMapFormat()).To(Succeed())
got, err := common.PropertyMapGet("scheduler-k3s", "node-js-app", "web.deployment")
Expect(err).NotTo(HaveOccurred())
Expect(got).To(Equal(map[string]string{"prometheus.io/scrape": "true"}))
}
func TestMigrateAnnotationsLabelsToMapFormatPreservesAlreadyMigrated(t *testing.T) {
setupAnnotationsLabelsMigrationTest(t, "node-js-app")
preserved := map[string]string{
"prometheus.io/scrape": "true",
"controller.config": "line one\nline two\nline three",
}
Expect(common.PropertyMapWrite("scheduler-k3s", "node-js-app", "web.deployment", preserved)).To(Succeed())
Expect(migrateAnnotationsLabelsToMapFormat()).To(Succeed())
got, err := common.PropertyMapGet("scheduler-k3s", "node-js-app", "web.deployment")
Expect(err).NotTo(HaveOccurred())
Expect(got).To(Equal(preserved))
}
func TestMigrateAnnotationsLabelsToMapFormatSkipsReservedPrefixes(t *testing.T) {
setupAnnotationsLabelsMigrationTest(t)
Expect(common.PropertyWrite("scheduler-k3s", "--global", "chart.cert-manager.deployment", "irrelevant")).To(Succeed())
Expect(common.PropertyWrite("scheduler-k3s", "--global", "node-profile-foo.json", "{}")).To(Succeed())
Expect(migrateAnnotationsLabelsToMapFormat()).To(Succeed())
Expect(common.PropertyGet("scheduler-k3s", "--global", "chart.cert-manager.deployment")).To(Equal("irrelevant"))
Expect(common.PropertyGet("scheduler-k3s", "--global", "node-profile-foo.json")).To(Equal("{}"))
}
func TestExitCodeError(t *testing.T) {
var _ common.ErrWithExitCode = (*ExitCodeError)(nil)
err := &ExitCodeError{Code: 3, Message: "boom"}
if err.ExitCode() != 3 {
t.Errorf("ExitCode() = %d, want 3", err.ExitCode())
}
if err.Error() != "boom" {
t.Errorf("Error() = %q, want %q", err.Error(), "boom")
}
}
func runPod(appName string, processType string, containerStatuses []corev1.ContainerStatus) v1.Pod {
return v1.Pod{
Status: v1.PodStatus{
Phase: v1.PodFailed,
ContainerStatuses: containerStatuses,
},
}
}
func terminatedContainerStatus(name string, exitCode int32, message string) corev1.ContainerStatus {
return corev1.ContainerStatus{
Name: name,
State: corev1.ContainerState{
Terminated: &corev1.ContainerStateTerminated{
ExitCode: exitCode,
Message: message,
},
},
}
}
func TestFailedRunContainerError(t *testing.T) {
containerName := fmt.Sprintf("%s-%s", "myapp", "run")
cases := []struct {
name string
appName string
processType string
statuses []corev1.ContainerStatus
wantCode int
wantError string
}{
{
name: "terminated run container with nonzero exit code",
appName: "myapp",
processType: "run",
statuses: []corev1.ContainerStatus{terminatedContainerStatus(containerName, 3, "")},
wantCode: 3,
wantError: "Unable to attach as the pod has already exited with a failed exit code: 3",
},
{
name: "terminated run container with nonzero exit code and message",
appName: "myapp",
processType: "run",
statuses: []corev1.ContainerStatus{terminatedContainerStatus(containerName, 137, "OOMKilled")},
wantCode: 137,
wantError: "Unable to attach as the pod has already exited with a failed exit code: 137 (OOMKilled)",
},
{
name: "terminated run container with zero exit code returns generic error",
appName: "myapp",
processType: "run",
statuses: []corev1.ContainerStatus{terminatedContainerStatus(containerName, 0, "")},
wantCode: -1,
wantError: "Unable to attach as the pod has already exited with a failed exit code",
},
{
name: "no container statuses returns generic error",
appName: "myapp",
processType: "run",
statuses: []corev1.ContainerStatus{},
wantCode: -1,
wantError: "Unable to attach as the pod has already exited with a failed exit code",
},
{
name: "container name mismatch returns generic error",
appName: "myapp",
processType: "run",
statuses: []corev1.ContainerStatus{terminatedContainerStatus("other-app-run", 3, "")},
wantCode: -1,
wantError: "Unable to attach as the pod has already exited with a failed exit code",
},
{
name: "unterminated matching container returns generic error",
appName: "myapp",
processType: "run",
statuses: []corev1.ContainerStatus{
{
Name: containerName,
State: corev1.ContainerState{},
},
},
wantCode: -1,
wantError: "Unable to attach as the pod has already exited with a failed exit code",
},
{
name: "unrelated terminated container before matching one",
appName: "myapp",
processType: "run",
statuses: []corev1.ContainerStatus{
terminatedContainerStatus("myapp-web", 9, ""),
terminatedContainerStatus(containerName, 5, ""),
},
wantCode: 5,
wantError: "Unable to attach as the pod has already exited with a failed exit code: 5",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := failedRunContainerError(tc.appName, tc.processType, runPod(tc.appName, tc.processType, tc.statuses))
if tc.wantCode == -1 {
if err == nil {
t.Fatal("failedRunContainerError() = nil, want generic error")
}
if _, ok := err.(*ExitCodeError); ok {
t.Fatalf("failedRunContainerError() returned %T, want generic error", err)
}
if err.Error() != tc.wantError {
t.Errorf("failedRunContainerError() error = %q, want %q", err.Error(), tc.wantError)
}
return
}
if err == nil {
t.Fatal("failedRunContainerError() = nil, want ExitCodeError")
}
exitErr, ok := err.(*ExitCodeError)
if !ok {
t.Fatalf("failedRunContainerError() returned %T, want *ExitCodeError", err)
}
if exitErr.ExitCode() != tc.wantCode {
t.Errorf("ExitCode() = %d, want %d", exitErr.ExitCode(), tc.wantCode)
}
if err.Error() != tc.wantError {
t.Errorf("Error() = %q, want %q", err.Error(), tc.wantError)
}
})
}
}