Concurrent commands for the same app could each read a setting, change it, and write it back, so all of them exited successfully while silently dropping changes made by the others. Updates to app settings are now serialized so every change is kept, and readers never see a partially written value.
1570 lines
46 KiB
Go
1570 lines
46 KiB
Go
package scheduler_k3s
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"github.com/dokku/dokku/plugins/common"
|
|
"github.com/dokku/dokku/plugins/config"
|
|
"github.com/dokku/dokku/plugins/cron"
|
|
nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts"
|
|
"github.com/dokku/dokku/plugins/registry"
|
|
"github.com/fatih/color"
|
|
"github.com/kballard/go-shellquote"
|
|
"github.com/ryanuber/columnize"
|
|
corev1 "k8s.io/api/core/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/kubectl/pkg/util/term"
|
|
"k8s.io/kubernetes/pkg/client/conditions"
|
|
)
|
|
|
|
// TriggerCorePostDeploy moves a configured kustomize root path to be in the app root dir
|
|
func TriggerCorePostDeploy(appName string) error {
|
|
return common.CorePostDeploy(common.CorePostDeployInput{
|
|
AppName: appName,
|
|
Destination: common.GetAppDataDirectory("scheduler-k3s", appName),
|
|
PluginName: "scheduler-k3s",
|
|
ExtractedPaths: []common.CorePostDeployPath{
|
|
{Path: "kustomization", IsDirectory: true},
|
|
},
|
|
})
|
|
}
|
|
|
|
// TriggerCorePostExtract moves a configured kustomize root path to be in the app root dir
|
|
// and warns about a missing cert issuer before any build work is performed
|
|
func TriggerCorePostExtract(appName string, sourceWorkDir string) error {
|
|
scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "")
|
|
globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local")
|
|
if scheduler == "" {
|
|
scheduler = globalScheduler
|
|
}
|
|
if scheduler == "k3s" {
|
|
warnMissingCertIssuer(context.Background(), appName)
|
|
}
|
|
|
|
destination := common.GetAppDataDirectory("scheduler-k3s", appName)
|
|
kustomizeRootPath := getComputedKustomizeRootPath(appName)
|
|
return common.CorePostExtract(common.CorePostExtractInput{
|
|
AppName: appName,
|
|
Destination: destination,
|
|
PluginName: "scheduler-k3s",
|
|
SourceWorkDir: sourceWorkDir,
|
|
ToExtract: []common.CorePostExtractToExtract{
|
|
{
|
|
Path: kustomizeRootPath,
|
|
IsDirectory: true,
|
|
Name: "config/kustomize",
|
|
Destination: "kustomization",
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
// TriggerInstall runs the install step for the scheduler-k3s plugin
|
|
func TriggerInstall() error {
|
|
if err := common.PropertySetup("scheduler-k3s"); err != nil {
|
|
return fmt.Errorf("Unable to install the scheduler-k3s plugin: %s", err.Error())
|
|
}
|
|
|
|
if err := common.SetupAppData("scheduler-k3s"); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := migrateChartPropertiesToMapFormat(); err != nil {
|
|
return fmt.Errorf("Unable to migrate chart properties: %w", err)
|
|
}
|
|
|
|
if err := migrateAnnotationsLabelsToMapFormat(); err != nil {
|
|
return fmt.Errorf("Unable to migrate annotations and labels: %w", err)
|
|
}
|
|
|
|
if err := syncExistingCertificates(); err != nil {
|
|
common.LogWarn(fmt.Sprintf("Warning: failed to sync existing certificates: %v", err))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// migrateChartPropertiesToMapFormat moves legacy per-key chart.<chart>.<key>
|
|
// property files into a single chart-overrides.<chart> JSON map per chart.
|
|
// Without this migration, a dokku upgrade would silently strand any existing
|
|
// chart overrides because the new render path only reads chart-overrides.*.
|
|
// Idempotent: once the legacy files are gone the helper short-circuits per
|
|
// chart, and legacy values take precedence over any partial pre-existing
|
|
// entries on re-runs.
|
|
func migrateChartPropertiesToMapFormat() error {
|
|
return common.PropertyLock("scheduler-k3s", "--global", func(p *common.LockedProperties) error {
|
|
for _, chart := range HelmCharts {
|
|
prefix := "chart." + chart.ReleaseName + "."
|
|
legacy, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", prefix)
|
|
if err != nil {
|
|
return fmt.Errorf("Unable to read legacy chart properties for %s: %w", chart.ReleaseName, err)
|
|
}
|
|
if len(legacy) == 0 {
|
|
continue
|
|
}
|
|
|
|
mapProperty := "chart-overrides." + chart.ReleaseName
|
|
merged, err := common.PropertyMapGet("scheduler-k3s", "--global", mapProperty)
|
|
if err != nil {
|
|
return fmt.Errorf("Unable to read existing chart-overrides for %s: %w", chart.ReleaseName, err)
|
|
}
|
|
for fullKey, value := range legacy {
|
|
merged[strings.TrimPrefix(fullKey, prefix)] = value
|
|
}
|
|
|
|
if err := p.MapWrite(mapProperty, merged); err != nil {
|
|
return fmt.Errorf("Unable to write chart-overrides for %s: %w", chart.ReleaseName, err)
|
|
}
|
|
|
|
for fullKey := range legacy {
|
|
if err := p.Delete(fullKey); err != nil {
|
|
return fmt.Errorf("Unable to remove legacy chart property %s: %w", fullKey, err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// migrateAnnotationsLabelsToMapFormat converts the legacy line-formatted
|
|
// annotation and label property files ("key: value" per line) into JSON maps
|
|
// written via PropertyMapWrite. The property name (e.g. "web.deployment",
|
|
// "labels.global.service") is unchanged - only the file content layout shifts.
|
|
// Without this migration, post-upgrade reads via PropertyMapGet would fail to
|
|
// parse the legacy line format as JSON. Idempotent: any property whose content
|
|
// is already JSON (or empty) is skipped by probing PropertyMapGet first.
|
|
func migrateAnnotationsLabelsToMapFormat() error {
|
|
annotationResources := map[string]bool{}
|
|
for _, rt := range AnnotationResourceTypes {
|
|
annotationResources[rt] = true
|
|
}
|
|
labelResources := map[string]bool{}
|
|
for _, rt := range LabelResourceTypes {
|
|
labelResources[rt] = true
|
|
}
|
|
|
|
scopes := []string{"--global"}
|
|
apps, err := common.UnfilteredDokkuApps()
|
|
if err != nil && !errors.Is(err, common.NoAppsExist) {
|
|
return fmt.Errorf("Unable to list apps for annotation/label migration: %w", err)
|
|
}
|
|
scopes = append(scopes, apps...)
|
|
|
|
for _, scope := range scopes {
|
|
properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", scope, "")
|
|
if err != nil {
|
|
return fmt.Errorf("Unable to list properties for %s: %w", scope, err)
|
|
}
|
|
|
|
for propertyName := range properties {
|
|
isLabel := strings.HasPrefix(propertyName, "labels.")
|
|
checkName := propertyName
|
|
if isLabel {
|
|
checkName = strings.TrimPrefix(propertyName, "labels.")
|
|
} else if isReservedAnnotationProperty(propertyName) {
|
|
continue
|
|
}
|
|
|
|
dot := strings.LastIndex(checkName, ".")
|
|
if dot <= 0 || dot == len(checkName)-1 {
|
|
continue
|
|
}
|
|
|
|
processType := checkName[:dot]
|
|
resourceType := checkName[dot+1:]
|
|
if processType == "" {
|
|
continue
|
|
}
|
|
|
|
if isLabel {
|
|
if !labelResources[resourceType] {
|
|
continue
|
|
}
|
|
} else {
|
|
if !annotationResources[resourceType] {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if err := migrateAnnotationLabelProperty(scope, propertyName); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// migrateAnnotationLabelProperty rewrites one legacy line-formatted property
|
|
// file as a JSON map. It is a no-op when the file is already valid JSON or
|
|
// empty (idempotent).
|
|
func migrateAnnotationLabelProperty(scope string, propertyName string) error {
|
|
return common.PropertyLock("scheduler-k3s", scope, func(p *common.LockedProperties) error {
|
|
if _, err := common.PropertyMapGet("scheduler-k3s", scope, propertyName); err == nil {
|
|
return nil
|
|
}
|
|
|
|
lines, err := common.PropertyListGet("scheduler-k3s", scope, propertyName)
|
|
if err != nil {
|
|
return fmt.Errorf("Unable to read legacy property %s/%s: %w", scope, propertyName, err)
|
|
}
|
|
|
|
m := map[string]string{}
|
|
for _, line := range lines {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
parts := strings.SplitN(line, ": ", 2)
|
|
if len(parts) != 2 {
|
|
return fmt.Errorf("Invalid legacy entry in %s/%s: %s", scope, propertyName, line)
|
|
}
|
|
m[parts[0]] = parts[1]
|
|
}
|
|
|
|
if err := p.MapWrite(propertyName, m); err != nil {
|
|
return fmt.Errorf("Unable to write %s/%s as map: %w", scope, propertyName, err)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// TriggerPostCertsUpdate handles post-certs-update trigger
|
|
func TriggerPostCertsUpdate(appName string) error {
|
|
scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "")
|
|
globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local")
|
|
if scheduler == "" {
|
|
scheduler = globalScheduler
|
|
}
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
common.LogInfo1(fmt.Sprintf("Syncing TLS certificate for %s to kubernetes", appName))
|
|
if err := CreateOrUpdateTLSSecret(ctx, appName); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !isAppDeployed(appName) {
|
|
return nil
|
|
}
|
|
|
|
imageTag, err := getDeployedAppImageTag(appName)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
common.LogInfo1(fmt.Sprintf("Triggering redeploy for %s to update ingress configuration", appName))
|
|
return TriggerSchedulerDeploy("k3s", appName, imageTag, "")
|
|
}
|
|
|
|
// TriggerPostCertsRemove handles post-certs-remove trigger
|
|
func TriggerPostCertsRemove(appName string) error {
|
|
scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "")
|
|
globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local")
|
|
if scheduler == "" {
|
|
scheduler = globalScheduler
|
|
}
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
deleted, err := DeleteTLSSecret(ctx, appName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !deleted {
|
|
return nil
|
|
}
|
|
|
|
if !isAppDeployed(appName) {
|
|
return nil
|
|
}
|
|
|
|
imageTag, err := getDeployedAppImageTag(appName)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
common.LogInfo1(fmt.Sprintf("Triggering redeploy for %s to update ingress configuration", appName))
|
|
return TriggerSchedulerDeploy("k3s", appName, imageTag, "")
|
|
}
|
|
|
|
// TriggerPostAppCloneSetup creates new scheduler-k3s files
|
|
func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error {
|
|
err := common.PropertyClone("scheduler-k3s", oldAppName, newAppName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// TriggerPostAppRenameSetup renames scheduler-k3s files
|
|
func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
|
|
if err := common.PropertyClone("scheduler-k3s", oldAppName, newAppName); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := common.PropertyDestroy("scheduler-k3s", oldAppName); err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
if _, err := DeleteTLSSecret(ctx, oldAppName); err != nil {
|
|
common.LogWarn(fmt.Sprintf("Error deleting TLS secret for old app name %s: %v", oldAppName, err))
|
|
}
|
|
|
|
if err := DeleteConfigSecret(ctx, oldAppName); err != nil {
|
|
common.LogWarn(fmt.Sprintf("Error deleting config secret for old app name %s: %v", oldAppName, err))
|
|
}
|
|
|
|
if err := DeleteImagePullSecret(ctx, oldAppName); err != nil {
|
|
common.LogWarn(fmt.Sprintf("Error deleting image pull secret for old app name %s: %v", oldAppName, err))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// TriggerPostCreate creates the scheduler-k3s data directory
|
|
func TriggerPostCreate(appName string) error {
|
|
return common.CreateAppDataDirectory("scheduler-k3s", appName)
|
|
}
|
|
|
|
// TriggerPostDelete destroys the scheduler-k3s data for a given app container
|
|
func TriggerPostDelete(appName string) error {
|
|
dataErr := common.RemoveAppDataDirectory("scheduler-k3s", appName)
|
|
propertyErr := common.PropertyDestroy("scheduler-k3s", appName)
|
|
|
|
if dataErr != nil {
|
|
return dataErr
|
|
}
|
|
|
|
return propertyErr
|
|
}
|
|
|
|
// TriggerSchedulerAppStatus returns the status of an app on the scheduler
|
|
func TriggerSchedulerAppStatus(scheduler string, appName string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
if err := clientset.Ping(); err != nil {
|
|
return fmt.Errorf("kubernetes api not available: %w", err)
|
|
}
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
deployments, err := clientset.ListDeployments(context.Background(), ListDeploymentsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error listing pods: %w", err)
|
|
}
|
|
|
|
processCount := 0
|
|
expectedProcesses := 0
|
|
runningProcesses := 0
|
|
for _, deployment := range deployments {
|
|
processCount += int(*deployment.Spec.Replicas)
|
|
expectedProcesses += int(*deployment.Spec.Replicas)
|
|
runningProcesses += int(deployment.Status.AvailableReplicas)
|
|
}
|
|
|
|
running := "true"
|
|
if expectedProcesses == 0 || runningProcesses == 0 {
|
|
running = "false"
|
|
} else if runningProcesses < expectedProcesses {
|
|
running = "mixed"
|
|
}
|
|
|
|
fmt.Printf("%d %s", processCount, running)
|
|
return nil
|
|
}
|
|
|
|
// TriggerSchedulerCronWrite writes out cron tasks for a given application
|
|
func TriggerSchedulerCronWrite(scheduler string, appName string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
if appName == "" {
|
|
return nil
|
|
}
|
|
|
|
cronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
|
|
if err != nil {
|
|
return fmt.Errorf("Error fetching cron tasks: %w", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
|
|
syscall.SIGINT,
|
|
syscall.SIGQUIT,
|
|
syscall.SIGTERM)
|
|
go func() {
|
|
<-signals
|
|
common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", appName))
|
|
cancel()
|
|
}()
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
for _, cronTask := range cronTasks {
|
|
labelSelector := []string{
|
|
fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
|
|
fmt.Sprintf("dokku.com/cron-hash=%s", cronIDLabelValue(cronTask.ID)),
|
|
}
|
|
|
|
if cronTask.Maintenance {
|
|
err = clientset.SuspendCronJobs(ctx, SuspendCronJobsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: strings.Join(labelSelector, ","),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error suspending cron jobs: %w", err)
|
|
}
|
|
} else {
|
|
err = clientset.ResumeCronJobs(ctx, ResumeCronJobsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: strings.Join(labelSelector, ","),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error resuming cron jobs: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// TriggerSchedulerDeploy deploys an image tag for a given application
|
|
func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string, processType string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
|
|
syscall.SIGINT,
|
|
syscall.SIGQUIT,
|
|
syscall.SIGTERM)
|
|
go func() {
|
|
<-signals
|
|
common.LogWarn(fmt.Sprintf("Deployment of %s has been cancelled", appName))
|
|
cancel()
|
|
}()
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
if err := createKubernetesNamespace(ctx, namespace); err != nil {
|
|
return fmt.Errorf("Error creating kubernetes namespace for deployment: %w", err)
|
|
}
|
|
|
|
helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating helm agent: %w", err)
|
|
}
|
|
|
|
priorValues := releaseValues(helmAgent, appName)
|
|
chartResult, err := BuildAppChart(ctx, appName, imageTag, BuildOptions{
|
|
FallbackImageMetadata: imageMetadataFromValues(priorValues),
|
|
RestartProcessType: processType,
|
|
PriorProcessDeploymentIDs: processDeploymentIDsFromValues(priorValues),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.RemoveAll(chartResult.ChartDir)
|
|
|
|
env, err := config.LoadMergedAppEnv(appName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error loading environment for deployment: %w", err)
|
|
}
|
|
|
|
globalAnnotations, err := getGlobalAnnotations(appName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting global annotations: %w", err)
|
|
}
|
|
|
|
globalLabels, err := getGlobalLabel(appName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting global labels: %w", err)
|
|
}
|
|
|
|
imagePullSecrets := getComputedImagePullSecrets(appName)
|
|
dokkuManagedPullSecret := false
|
|
var dokkuPullSecretBytes []byte
|
|
if imagePullSecrets == "" {
|
|
dockerConfigPath := filepath.Join(registry.GetComputedAppRegistryConfigDir(appName), "config.json")
|
|
if fi, err := os.Stat(dockerConfigPath); err == nil && !fi.IsDir() {
|
|
b, err := os.ReadFile(dockerConfigPath)
|
|
if err != nil {
|
|
return fmt.Errorf("Error reading docker config: %w", err)
|
|
}
|
|
|
|
imagePullSecrets = GetImagePullSecretName(appName)
|
|
dokkuManagedPullSecret = true
|
|
dokkuPullSecretBytes = b
|
|
}
|
|
}
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
ingresses, err := clientset.ListIngresses(ctx, ListIngressesInput{
|
|
Namespace: chartResult.Namespace,
|
|
LabelSelector: fmt.Sprintf("app.kubernetes.io/instance=%s-web", appName),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error listing ingresses: %w", err)
|
|
}
|
|
|
|
ingressesToDelete := []string{}
|
|
|
|
for _, ingress := range ingresses {
|
|
ingressIngressMethod := ingress.Annotations["dokku.com/ingress-method"]
|
|
if ingressIngressMethod != "domains" {
|
|
ingressesToDelete = append(ingressesToDelete, ingress.Name)
|
|
}
|
|
}
|
|
|
|
if len(ingressesToDelete) > 0 {
|
|
common.LogWarn("Manually removing non-matching ingress resources")
|
|
}
|
|
for _, ingressName := range ingressesToDelete {
|
|
common.LogVerboseQuiet(fmt.Sprintf("Removing non-matching ingress resource: %s", ingressName))
|
|
err := clientset.DeleteIngress(ctx, DeleteIngressInput{
|
|
Name: ingressName,
|
|
Namespace: chartResult.Namespace,
|
|
})
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("Error deleting ingress: %w", err)
|
|
}
|
|
}
|
|
|
|
if err := CreateOrUpdateConfigSecret(ctx, CreateOrUpdateConfigSecretInput{
|
|
AppName: appName,
|
|
Env: env.Map(),
|
|
Annotations: globalAnnotations.SecretAnnotations,
|
|
Labels: globalLabels.SecretLabels,
|
|
}); err != nil {
|
|
return fmt.Errorf("Error syncing config secret: %w", err)
|
|
}
|
|
|
|
if dokkuManagedPullSecret {
|
|
if err := CreateOrUpdateImagePullSecret(ctx, CreateOrUpdateImagePullSecretInput{
|
|
AppName: appName,
|
|
DockerConfigJSON: dokkuPullSecretBytes,
|
|
Annotations: globalAnnotations.SecretAnnotations,
|
|
Labels: globalLabels.SecretLabels,
|
|
}); err != nil {
|
|
return fmt.Errorf("Error syncing image pull secret: %w", err)
|
|
}
|
|
} else {
|
|
if err := DeleteImagePullSecret(ctx, appName); err != nil {
|
|
return fmt.Errorf("Error removing stale image pull secret: %w", err)
|
|
}
|
|
}
|
|
|
|
keepImagePullSecrets := []string{}
|
|
if imagePullSecrets != "" {
|
|
keepImagePullSecrets = []string{imagePullSecrets}
|
|
}
|
|
if err := pruneStaleImagePullSecretsFromDeployments(ctx, clientset, chartResult.Namespace, appName, keepImagePullSecrets); err != nil {
|
|
return fmt.Errorf("Error pruning stale imagePullSecrets entries: %w", err)
|
|
}
|
|
|
|
common.LogInfo2(fmt.Sprintf("Installing %s", appName))
|
|
err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
|
|
ChartPath: chartResult.ChartPath,
|
|
KustomizeRootPath: chartResult.KustomizeRootPath,
|
|
Namespace: chartResult.Namespace,
|
|
ReleaseName: chartResult.ReleaseName,
|
|
RollbackOnFailure: chartResult.RollbackOnFailure,
|
|
Timeout: chartResult.Timeout,
|
|
Wait: true,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
common.LogInfo1("Running post-deploy")
|
|
_, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
|
|
Args: []string{appName, "", "", imageTag},
|
|
StreamStdio: true,
|
|
Trigger: "core-post-deploy",
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error running core-post-deploy: %w", err)
|
|
|
|
}
|
|
_, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
|
|
Args: []string{appName, "", "", imageTag},
|
|
StreamStdio: true,
|
|
Trigger: "post-deploy",
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error running post-deploy: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// TriggerSchedulerIsDeployed returns true if given app has a running container
|
|
func TriggerSchedulerIsDeployed(scheduler string, appName string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
if isAppDeployed(appName) {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("App %s is not deployed", appName)
|
|
}
|
|
|
|
// TriggerSchedulerUsesHostCron reports that the k3s scheduler does not use the
|
|
// host crontab; it manages its own in-cluster CronJobs
|
|
func TriggerSchedulerUsesHostCron(scheduler string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
fmt.Println("false")
|
|
return nil
|
|
}
|
|
|
|
// TriggerSchedulerEnter enters a container for a given application
|
|
func TriggerSchedulerEnter(scheduler string, appName string, processType string, podName string, args []string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
|
|
syscall.SIGINT,
|
|
syscall.SIGQUIT,
|
|
syscall.SIGTERM)
|
|
go func() {
|
|
<-signals
|
|
cancel()
|
|
}()
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
if err := clientset.Ping(); err != nil {
|
|
return fmt.Errorf("kubernetes api not available: %w", err)
|
|
}
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
|
|
processIndex := 1
|
|
if processType != "" {
|
|
parts := strings.SplitN(processType, ".", 2)
|
|
if len(parts) == 2 {
|
|
processType = parts[0]
|
|
processIndex, err = strconv.Atoi(parts[1])
|
|
if err != nil {
|
|
return fmt.Errorf("Error parsing process index: %w", err)
|
|
}
|
|
}
|
|
labelSelector = append(labelSelector, fmt.Sprintf("app.kubernetes.io/name=%s", processType))
|
|
}
|
|
|
|
pods, err := clientset.ListPods(ctx, ListPodsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: strings.Join(labelSelector, ","),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error listing pods: %w", err)
|
|
}
|
|
|
|
if len(pods) == 0 {
|
|
return fmt.Errorf("No pods found for app %s", appName)
|
|
}
|
|
|
|
processIndex--
|
|
if processIndex > len(pods) {
|
|
return fmt.Errorf("Process index %d out of range for app %s", processIndex, appName)
|
|
}
|
|
|
|
var selectedPod corev1.Pod
|
|
if podName != "" {
|
|
for _, pod := range pods {
|
|
if pod.Name == podName {
|
|
selectedPod = pod
|
|
break
|
|
}
|
|
}
|
|
} else {
|
|
selectedPod = pods[processIndex]
|
|
}
|
|
|
|
command := args
|
|
if len(args) == 0 {
|
|
command = []string{common.GetDokkuAppShell(appName)}
|
|
}
|
|
|
|
entrypoint := ""
|
|
if selectedPod.Annotations["dokku.com/builder-type"] == "herokuish" {
|
|
entrypoint = "/exec"
|
|
}
|
|
|
|
return enterPod(ctx, EnterPodInput{
|
|
Clientset: clientset,
|
|
Command: command,
|
|
Entrypoint: entrypoint,
|
|
SelectedPod: selectedPod,
|
|
WaitTimeout: 10,
|
|
})
|
|
}
|
|
|
|
// TriggerSchedulerLogs displays logs for a given application
|
|
func TriggerSchedulerLogs(scheduler string, appName string, processType string, tail bool, quiet bool, numLines int64) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
return clientset.StreamLogs(context.Background(), StreamLogsInput{
|
|
Namespace: getComputedNamespace(appName),
|
|
ContainerName: processType,
|
|
LabelSelector: []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)},
|
|
TailLines: numLines,
|
|
Follow: tail,
|
|
Quiet: quiet,
|
|
})
|
|
}
|
|
|
|
// TriggerSchedulerProxyConfig displays nginx config for a given application
|
|
func TriggerSchedulerProxyConfig(scheduler string, appName string, proxyType string) error {
|
|
if scheduler != "k3s" || proxyType != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
pods, err := clientset.ListPods(context.Background(), ListPodsInput{
|
|
Namespace: "ingress-nginx",
|
|
LabelSelector: "app.kubernetes.io/name=ingress-nginx",
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error listing pods: %w", err)
|
|
}
|
|
|
|
if len(pods) == 0 {
|
|
return errors.New("No pods found for ingress-nginx")
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
w := io.MultiWriter(&buf)
|
|
|
|
command := []string{"cat", "/etc/nginx/nginx.conf"}
|
|
err = clientset.ExecCommand(context.Background(), ExecCommandInput{
|
|
Command: command,
|
|
ContainerName: "controller",
|
|
Name: pods[0].Name,
|
|
Namespace: "ingress-nginx",
|
|
Stdout: w,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error reading nginx config: %w", err)
|
|
}
|
|
|
|
// split buffer string into lines
|
|
lines := strings.Split(buf.String(), "\n")
|
|
|
|
// get every domain block from the nginx config
|
|
// domain blocks begin with "## start server DOMAIN_NAME" and end with "## end server DOMAIN_NAME"
|
|
// each line also may have space characters at the beginning
|
|
domainBlocks := map[string][]string{}
|
|
for i, line := range lines {
|
|
strippedLine := strings.TrimSpace(line)
|
|
if strings.HasPrefix(strippedLine, "## start server ") {
|
|
domainName := strings.TrimPrefix(strippedLine, "## start server ")
|
|
domainBlocks[domainName] = []string{}
|
|
for _, nextLine := range lines[i+1:] {
|
|
nextStrippedLine := strings.TrimSpace(nextLine)
|
|
if strings.HasPrefix(nextStrippedLine, "## end server "+domainName) {
|
|
break
|
|
}
|
|
domainBlocks[domainName] = append(domainBlocks[domainName], nextLine)
|
|
}
|
|
}
|
|
}
|
|
|
|
// get all domains for this app
|
|
domains := []string{}
|
|
_, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
|
|
Trigger: "domains-vhost-enabled",
|
|
Args: []string{appName},
|
|
StreamStdio: true,
|
|
})
|
|
if err == nil {
|
|
results, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
|
|
Trigger: "domains-list",
|
|
Args: []string{appName},
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting domains for deployment: %w", err)
|
|
}
|
|
|
|
for _, domain := range strings.Split(results.StdoutContents(), "\n") {
|
|
domain = strings.TrimSpace(domain)
|
|
if domain != "" {
|
|
domains = append(domains, domain)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, domain := range domains {
|
|
if domainBlock, ok := domainBlocks[domain]; ok {
|
|
fmt.Println(strings.Join(domainBlock, "\n"))
|
|
}
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
// TriggerSchedulerProxyLogs displays nginx logs for a given application
|
|
func TriggerSchedulerProxyLogs(scheduler string, appName string, proxyType string, logType string, tail bool, numLines int64) error {
|
|
if scheduler != "k3s" || proxyType != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
filename := ""
|
|
if logType == "access" {
|
|
filename = nginxvhosts.ComputedAccessLogPath(appName)
|
|
} else if logType == "error" {
|
|
filename = nginxvhosts.ComputedErrorLogPath(appName)
|
|
} else {
|
|
return errors.New("Invalid log type")
|
|
}
|
|
|
|
command := []string{"tail"}
|
|
if tail {
|
|
command = append(command, "-F")
|
|
}
|
|
if numLines > 0 {
|
|
command = append(command, "-n")
|
|
command = append(command, strconv.FormatInt(numLines, 10))
|
|
}
|
|
command = append(command, filename)
|
|
|
|
pods, err := clientset.ListPods(context.Background(), ListPodsInput{
|
|
Namespace: "ingress-nginx",
|
|
LabelSelector: "app.kubernetes.io/name=ingress-nginx",
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error listing pods: %w", err)
|
|
}
|
|
|
|
if len(pods) == 0 {
|
|
return errors.New("No pods found for ingress-nginx")
|
|
}
|
|
|
|
return clientset.ExecCommand(context.Background(), ExecCommandInput{
|
|
Command: command,
|
|
ContainerName: "controller",
|
|
Name: pods[0].Name,
|
|
Namespace: "ingress-nginx",
|
|
})
|
|
}
|
|
|
|
// ExitCodeError carries the exit code of a failed user command so that the
|
|
// scheduler trigger can exit with the command's own status rather than a
|
|
// generic failure. It implements common.ErrWithExitCode, which
|
|
// common.LogFailWithError honors, matching docker-local scheduler behavior.
|
|
type ExitCodeError struct {
|
|
// Code is the exit code of the failed command or container
|
|
Code int
|
|
|
|
// Message is the error text printed to stderr
|
|
Message string
|
|
}
|
|
|
|
var _ common.ErrWithExitCode = (*ExitCodeError)(nil)
|
|
|
|
// Error returns the error text to print to stderr
|
|
func (e *ExitCodeError) Error() string {
|
|
return e.Message
|
|
}
|
|
|
|
// ExitCode returns the exit code to use in case this error bubbles up into an
|
|
// os.Exit() call
|
|
func (e *ExitCodeError) ExitCode() int {
|
|
return e.Code
|
|
}
|
|
|
|
// failedRunContainerError builds an ExitCodeError from the terminated run
|
|
// container of a failed run pod. The container's exit code is the exit code of
|
|
// the command the run pod executed, so it can be propagated to the caller.
|
|
// When no terminated container with a nonzero exit code exists, a generic
|
|
// failure error is returned instead.
|
|
func failedRunContainerError(appName string, processType string, pod v1.Pod) error {
|
|
containerName := fmt.Sprintf("%s-%s", appName, processType)
|
|
for _, status := range pod.Status.ContainerStatuses {
|
|
if status.Name != containerName || status.State.Terminated == nil {
|
|
continue
|
|
}
|
|
exitCode := int(status.State.Terminated.ExitCode)
|
|
if exitCode > 0 {
|
|
message := fmt.Sprintf("Unable to attach as the pod has already exited with a failed exit code: %d", exitCode)
|
|
if messageDetail := status.State.Terminated.Message; messageDetail != "" {
|
|
message = fmt.Sprintf("%s (%s)", message, messageDetail)
|
|
}
|
|
return &ExitCodeError{Code: exitCode, Message: message}
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("Unable to attach as the pod has already exited with a failed exit code")
|
|
}
|
|
|
|
// TriggerSchedulerRun runs a command in an ephemeral container
|
|
func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
extraEnv := map[string]string{}
|
|
if envCount > 0 {
|
|
var envPairs []string
|
|
envPairs, args = args[0:envCount], args[envCount:]
|
|
for _, envPair := range envPairs {
|
|
parts := strings.SplitN(envPair, "=", 2)
|
|
if len(parts) != 2 {
|
|
return fmt.Errorf("Invalid environment variable pair: %s", envPair)
|
|
}
|
|
extraEnv[parts[0]] = parts[1]
|
|
}
|
|
}
|
|
|
|
imageTag, err := common.GetRunningImageTag(appName, "")
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting running image tag: %w", err)
|
|
}
|
|
image, err := common.ResolveDeployingAppImageName(appName, imageTag, "")
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting deploying app image name: %w", err)
|
|
}
|
|
|
|
// The image-stage guard exists to catch a build-stage image being run before
|
|
// the app was ever deployed. It can only be answered from a local copy of the
|
|
// image, which a k3s host is free to reap since the cluster pulls its own. In
|
|
// that case the guard is skipped: resolveImageMetadata below falls back to the
|
|
// app's Helm release, and the existence of that release is itself proof the
|
|
// app got past the release stage.
|
|
if common.VerifyImage(image) {
|
|
imageStage, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.image-stage\" }}")
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting image stage: %w", err)
|
|
}
|
|
if imageStage != "release" {
|
|
common.LogWarn(fmt.Sprintf("Invalid image stage detected: expected 'release', got '%s'", imageStage))
|
|
return fmt.Errorf("Successfully deploy your app to fix dokku run calls")
|
|
}
|
|
}
|
|
|
|
dokkuRmContainer := os.Getenv("DOKKU_RM_CONTAINER")
|
|
if dokkuRmContainer == "" {
|
|
resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
|
|
Trigger: "config-get",
|
|
Args: []string{appName, "DOKKU_RM_CONTAINER"},
|
|
})
|
|
if err != nil {
|
|
resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
|
|
Trigger: "config-get-global",
|
|
Args: []string{"DOKKU_RM_CONTAINER"},
|
|
})
|
|
if err == nil {
|
|
dokkuRmContainer = strings.TrimSpace(resp.Stdout)
|
|
}
|
|
} else {
|
|
dokkuRmContainer = strings.TrimSpace(resp.Stdout)
|
|
}
|
|
}
|
|
if dokkuRmContainer == "" {
|
|
dokkuRmContainer = "true"
|
|
}
|
|
|
|
rmContainer, err := strconv.ParseBool(dokkuRmContainer)
|
|
if err != nil {
|
|
return fmt.Errorf("Error parsing DOKKU_RM_CONTAINER value as boolean: %w", err)
|
|
}
|
|
|
|
labels := map[string]string{
|
|
"app.kubernetes.io/part-of": appName,
|
|
}
|
|
annotations := map[string]string{}
|
|
|
|
if os.Getenv("DOKKU_TRACE") == "1" {
|
|
extraEnv["TRACE"] = "true"
|
|
}
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
if err := clientset.Ping(); err != nil {
|
|
return fmt.Errorf("kubernetes api not available: %w", err)
|
|
}
|
|
|
|
processType := "run"
|
|
if os.Getenv("DOKKU_CRON_ID") != "" {
|
|
processType = "cron"
|
|
cronID := os.Getenv("DOKKU_CRON_ID")
|
|
cronHash := cronIDLabelValue(cronID)
|
|
labels["dokku.com/cron-hash"] = cronHash
|
|
annotations["dokku.com/cron-hash"] = cronHash
|
|
annotations["dokku.com/cron-id"] = cronID
|
|
concurrencyPolicy := strings.ToUpper(os.Getenv("DOKKU_CONCURRENCY_POLICY"))
|
|
switch concurrencyPolicy {
|
|
case "forbid":
|
|
pods, err := clientset.ListPods(context.Background(), ListPodsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: fmt.Sprintf("dokku.com/cron-hash=%s", cronHash),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error listing pods: %w", err)
|
|
}
|
|
if len(pods) > 0 {
|
|
return fmt.Errorf("There is a running pod with the same dokku.com/cron-hash label")
|
|
}
|
|
case "replace":
|
|
err := clientset.DeletePod(context.Background(), DeletePodInput{
|
|
Namespace: namespace,
|
|
LabelSelector: fmt.Sprintf("dokku.com/cron-hash=%s", cronHash),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error deleting pod: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating helm agent: %w", err)
|
|
}
|
|
|
|
values := releaseValues(helmAgent, appName)
|
|
imageMetadata, err := resolveImageMetadata(appName, image, imageMetadataFromValues(values))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
imageSourceType := imageMetadata.SourceType
|
|
|
|
// todo: do something with docker args
|
|
command := args
|
|
commandShell := common.GetDokkuAppShell(appName)
|
|
if len(args) == 0 {
|
|
command = []string{commandShell}
|
|
} else if len(args) == 1 {
|
|
resp, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
|
|
Trigger: "procfile-get-command",
|
|
Args: []string{appName, args[0], "5000"},
|
|
})
|
|
if err == nil && resp.StdoutContents() != "" {
|
|
common.LogInfo1Quiet(fmt.Sprintf("Found '%s' in Procfile, running that command", args[0]))
|
|
words, err := shellquote.Split(resp.StdoutContents())
|
|
if err != nil {
|
|
return fmt.Errorf("Error parsing Procfile command: %w", err)
|
|
}
|
|
command = words
|
|
}
|
|
}
|
|
|
|
entrypoint := ""
|
|
switch imageSourceType {
|
|
case "herokuish":
|
|
entrypoint = "/exec"
|
|
case "pack":
|
|
entrypoint = "launcher"
|
|
}
|
|
|
|
globalValues, ok := values["global"].(map[string]interface{})
|
|
if !ok {
|
|
return errors.New("Global helm values not found")
|
|
}
|
|
|
|
deploymentIDValue, ok := globalValues["deployment_id"].(string)
|
|
if !ok {
|
|
return fmt.Errorf("Deployment ID is not a string")
|
|
}
|
|
if len(deploymentIDValue) == 0 {
|
|
return fmt.Errorf("Deployment ID is empty")
|
|
}
|
|
deploymentID, err := strconv.ParseInt(deploymentIDValue, 10, 64)
|
|
if err != nil {
|
|
return fmt.Errorf("Error parsing deployment ID: %w", err)
|
|
}
|
|
|
|
attachToPod := os.Getenv("DOKKU_DETACH_CONTAINER") != "1"
|
|
|
|
forceTTY := common.ToBool(os.Getenv("DOKKU_FORCE_TTY"))
|
|
disableTTY := common.ToBool(os.Getenv("DOKKU_DISABLE_TTY"))
|
|
ttyProbe := term.TTY{Out: os.Stdout}
|
|
hasTTY := ttyProbe.MonitorSize(ttyProbe.GetSize()) != nil
|
|
streamLogsOnly := attachToPod && !forceTTY && (disableTTY || !hasTTY)
|
|
|
|
imagePullSecrets := getComputedImagePullSecrets(appName)
|
|
if imagePullSecrets == "" {
|
|
imagePullSecrets = GetImagePullSecretName(appName)
|
|
_, err := clientset.GetSecret(context.Background(), GetSecretInput{
|
|
Name: imagePullSecrets,
|
|
Namespace: namespace,
|
|
})
|
|
|
|
if err != nil {
|
|
if _, ok := err.(*NotFoundError); !ok {
|
|
return fmt.Errorf("Error getting image pull secret: %w", err)
|
|
}
|
|
imagePullSecrets = ""
|
|
}
|
|
}
|
|
|
|
securityContext, err := getSecurityContext(appName, "run")
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting security context: %w", err)
|
|
}
|
|
|
|
activeDeadlineSeconds := int64(86400) // 24 hours
|
|
if os.Getenv("DOKKU_RUN_TTL_SECONDS") != "" {
|
|
activeDeadlineSeconds, err = strconv.ParseInt(os.Getenv("DOKKU_RUN_TTL_SECONDS"), 10, 64)
|
|
if err != nil {
|
|
return fmt.Errorf("Error parsing DOKKU_RUN_TTL_SECONDS value as int64: %w", err)
|
|
}
|
|
}
|
|
|
|
workingDir := imageMetadata.WorkingDir
|
|
job, err := templateKubernetesJob(Job{
|
|
ActiveDeadlineSeconds: activeDeadlineSeconds,
|
|
Annotations: annotations,
|
|
AppName: appName,
|
|
Command: command,
|
|
DeploymentID: deploymentID,
|
|
Entrypoint: entrypoint,
|
|
Env: extraEnv,
|
|
Image: image,
|
|
ImagePullSecrets: imagePullSecrets,
|
|
ImageSourceType: imageSourceType,
|
|
Interactive: attachToPod || common.ToBool(os.Getenv("DOKKU_FORCE_TTY")),
|
|
Labels: labels,
|
|
Namespace: namespace,
|
|
ProcessType: processType,
|
|
RemoveContainer: rmContainer,
|
|
SecurityContext: securityContext,
|
|
WorkingDir: workingDir,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error templating job: %w", err)
|
|
}
|
|
|
|
if os.Getenv("FORCE_TTY") == "1" {
|
|
color.NoColor = false
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
|
|
syscall.SIGINT,
|
|
syscall.SIGQUIT,
|
|
syscall.SIGTERM)
|
|
go func() {
|
|
<-signals
|
|
if attachToPod {
|
|
clientset.DeleteJob(ctx, DeleteJobInput{ // nolint: errcheck
|
|
Name: job.Name,
|
|
Namespace: namespace,
|
|
})
|
|
}
|
|
cancel()
|
|
}()
|
|
|
|
createdJob, err := clientset.CreateJob(ctx, CreateJobInput{
|
|
Job: job,
|
|
Namespace: namespace,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating job: %w", err)
|
|
}
|
|
|
|
if attachToPod {
|
|
defer func() {
|
|
clientset.DeleteJob(ctx, DeleteJobInput{ // nolint: errcheck
|
|
Name: job.Name,
|
|
Namespace: namespace,
|
|
})
|
|
}()
|
|
}
|
|
|
|
batchJobSelector := fmt.Sprintf("batch.kubernetes.io/job-name=%s", createdJob.Name)
|
|
pods, err := waitForPodToExist(ctx, WaitForPodToExistInput{
|
|
Clientset: clientset,
|
|
Namespace: namespace,
|
|
RetryCount: 3,
|
|
LabelSelector: batchJobSelector,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error waiting for pod to exist: %w", err)
|
|
}
|
|
if !attachToPod {
|
|
fmt.Println(pods[0].Name)
|
|
return nil
|
|
}
|
|
|
|
err = waitForPodBySelectorRunning(ctx, WaitForPodBySelectorRunningInput{
|
|
Clientset: clientset,
|
|
Namespace: namespace,
|
|
LabelSelector: batchJobSelector,
|
|
Timeout: 30,
|
|
Waiter: isPodReady,
|
|
})
|
|
if streamLogsOnly && (err == nil || errors.Is(err, conditions.ErrPodCompleted)) {
|
|
if streamErr := clientset.StreamLogs(ctx, StreamLogsInput{
|
|
Follow: true,
|
|
LabelSelector: []string{batchJobSelector},
|
|
Namespace: namespace,
|
|
Quiet: true,
|
|
}); streamErr != nil {
|
|
return fmt.Errorf("Error streaming logs: %w", streamErr)
|
|
}
|
|
|
|
selectedPod, waitErr := waitForPodBySelectorCompleted(ctx, WaitForPodBySelectorCompletedInput{
|
|
Clientset: clientset,
|
|
Namespace: namespace,
|
|
LabelSelector: batchJobSelector,
|
|
Timeout: 10,
|
|
})
|
|
if waitErr != nil {
|
|
return fmt.Errorf("Pod did not reach terminal state after logs streamed: %w", waitErr)
|
|
}
|
|
switch selectedPod.Status.Phase {
|
|
case v1.PodSucceeded:
|
|
return nil
|
|
case v1.PodFailed:
|
|
return failedRunContainerError(appName, processType, selectedPod)
|
|
}
|
|
return fmt.Errorf("Unable to attach as the pod is in an unknown state: %s", selectedPod.Status.Phase)
|
|
}
|
|
if err != nil {
|
|
if errors.Is(err, conditions.ErrPodCompleted) {
|
|
pods, podErr := clientset.ListPods(ctx, ListPodsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: batchJobSelector,
|
|
})
|
|
if podErr != nil {
|
|
return fmt.Errorf("Error completed pod: %w", err)
|
|
}
|
|
selectedPod := pods[0]
|
|
|
|
err := clientset.StreamLogs(ctx, StreamLogsInput{
|
|
ContainerName: processType,
|
|
Follow: false,
|
|
LabelSelector: []string{batchJobSelector},
|
|
Namespace: namespace,
|
|
Quiet: true,
|
|
SinceSeconds: 10,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error streaming logs: %w", err)
|
|
}
|
|
if selectedPod.Status.Phase == v1.PodFailed {
|
|
return failedRunContainerError(appName, processType, selectedPod)
|
|
} else if selectedPod.Status.Phase == v1.PodSucceeded {
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("Error waiting for pod to be running: %w", err)
|
|
}
|
|
|
|
pods, err = clientset.ListPods(ctx, ListPodsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: batchJobSelector,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting pod: %w", err)
|
|
}
|
|
selectedPod := pods[0]
|
|
|
|
switch selectedPod.Status.Phase {
|
|
case v1.PodFailed, v1.PodSucceeded:
|
|
if selectedPod.Status.Phase == v1.PodFailed {
|
|
return failedRunContainerError(appName, processType, selectedPod)
|
|
} else if selectedPod.Status.Phase == v1.PodSucceeded {
|
|
return errors.New("Unable to attach as the pod has already exited with a successful exit code")
|
|
}
|
|
case v1.PodRunning:
|
|
return enterPod(ctx, EnterPodInput{
|
|
AllowCompletion: true,
|
|
Clientset: clientset,
|
|
Command: command,
|
|
Entrypoint: entrypoint,
|
|
SelectedPod: selectedPod,
|
|
WaitTimeout: 10,
|
|
})
|
|
default:
|
|
return fmt.Errorf("Unable to attach as the pod is in an unknown state: %s", selectedPod.Status.Phase)
|
|
}
|
|
// todo: support scheduler-post-run
|
|
|
|
return nil
|
|
}
|
|
|
|
// TriggerSchedulerRunList lists one-off run pods for a given application
|
|
func TriggerSchedulerRunList(scheduler string, appName string, format string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
|
|
syscall.SIGINT,
|
|
syscall.SIGQUIT,
|
|
syscall.SIGTERM)
|
|
go func() {
|
|
<-signals
|
|
cancel()
|
|
}()
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
if err := clientset.Ping(); err != nil {
|
|
return fmt.Errorf("kubernetes api not available: %w", err)
|
|
}
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
|
|
LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
|
|
Namespace: namespace,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error getting cron jobs: %w", err)
|
|
}
|
|
|
|
type CronJobTask struct {
|
|
ID string `json:"id"`
|
|
AppName string `json:"app"`
|
|
Command string `json:"command"`
|
|
Schedule string `json:"schedule"`
|
|
}
|
|
|
|
data := []CronJobTask{}
|
|
lines := []string{"ID | Schedule | Command"}
|
|
for _, cronJob := range cronJobs {
|
|
command := ""
|
|
for _, container := range cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers {
|
|
if container.Name == fmt.Sprintf("%s-cron", appName) {
|
|
command = strings.Join(container.Args, " ")
|
|
}
|
|
}
|
|
|
|
cronID, ok := cronJob.Annotations["dokku.com/cron-id"]
|
|
if !ok {
|
|
common.LogWarn(fmt.Sprintf("Cron job %s does not have a dokku.com/cron-id annotation", cronJob.Name))
|
|
continue
|
|
}
|
|
|
|
lines = append(lines, fmt.Sprintf("%s | %s | %s", cronID, cronJob.Spec.Schedule, command))
|
|
data = append(data, CronJobTask{
|
|
ID: cronID,
|
|
AppName: appName,
|
|
Command: command,
|
|
Schedule: cronJob.Spec.Schedule,
|
|
})
|
|
}
|
|
|
|
if format == "stdout" {
|
|
result := columnize.SimpleFormat(lines)
|
|
fmt.Println(result)
|
|
} else {
|
|
b, err := json.Marshal(data)
|
|
if err != nil {
|
|
return fmt.Errorf("Error marshalling cron jobs: %w", err)
|
|
}
|
|
fmt.Println(string(b))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// TriggerSchedulerPostDelete destroys the scheduler-k3s data for a given app container
|
|
func TriggerSchedulerPostDelete(scheduler string, appName string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
dataErr := common.RemoveAppDataDirectory("logs", appName)
|
|
propertyErr := common.PropertyDestroy("logs", appName)
|
|
|
|
if dataErr != nil {
|
|
return dataErr
|
|
}
|
|
|
|
if propertyErr != nil {
|
|
return propertyErr
|
|
}
|
|
|
|
if isK3sKubernetes() {
|
|
if err := isK3sInstalled(); err != nil {
|
|
common.LogWarn("k3s is not installed, skipping")
|
|
return nil
|
|
}
|
|
}
|
|
|
|
if err := isKubernetesAvailable(); err != nil {
|
|
return fmt.Errorf("kubernetes api not available: %w", err)
|
|
}
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating helm agent: %w", err)
|
|
}
|
|
|
|
err = helmAgent.UninstallChart(appName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error uninstalling chart: %w", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
if _, err := DeleteTLSSecret(ctx, appName); err != nil {
|
|
common.LogWarn(fmt.Sprintf("Error deleting TLS secret for %s: %v", appName, err))
|
|
}
|
|
|
|
if err := DeleteConfigSecret(ctx, appName); err != nil {
|
|
common.LogWarn(fmt.Sprintf("Error deleting config secret for %s: %v", appName, err))
|
|
}
|
|
|
|
if err := DeleteImagePullSecret(ctx, appName); err != nil {
|
|
common.LogWarn(fmt.Sprintf("Error deleting image pull secret for %s: %v", appName, err))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// TriggerSchedulerStop stops an application
|
|
func TriggerSchedulerStop(scheduler string, appName string) error {
|
|
if scheduler != "k3s" {
|
|
return nil
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
|
|
syscall.SIGINT,
|
|
syscall.SIGQUIT,
|
|
syscall.SIGTERM)
|
|
go func() {
|
|
<-signals
|
|
cancel()
|
|
}()
|
|
|
|
clientset, err := NewKubernetesClient()
|
|
if err != nil {
|
|
if isK3sKubernetes() {
|
|
if err := isK3sInstalled(); err != nil {
|
|
common.LogWarn("k3s is not installed, skipping")
|
|
return nil
|
|
}
|
|
}
|
|
return fmt.Errorf("Error creating kubernetes client: %w", err)
|
|
}
|
|
|
|
if err := clientset.Ping(); err != nil {
|
|
return fmt.Errorf("kubernetes api not available: %w", err)
|
|
}
|
|
|
|
namespace := getComputedNamespace(appName)
|
|
deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error listing deployments: %w", err)
|
|
}
|
|
|
|
for _, deployment := range deployments {
|
|
processType, ok := deployment.Labels["app.kubernetes.io/name"]
|
|
if !ok {
|
|
return fmt.Errorf("Deployment %s does not have a process type label", deployment.Name)
|
|
}
|
|
common.LogVerboseQuiet(fmt.Sprintf("Stopping %s process", processType))
|
|
err := clientset.ScaleDeployment(ctx, ScaleDeploymentInput{
|
|
Name: deployment.Name,
|
|
Namespace: namespace,
|
|
Replicas: 0,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error updating deployment scale: %w", err)
|
|
}
|
|
}
|
|
|
|
// get all cronjobs for the app
|
|
err = clientset.SuspendCronJobs(ctx, SuspendCronJobsInput{
|
|
Namespace: namespace,
|
|
LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Error suspending cron jobs: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|