diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index f1c66cee5..c1c4e705c 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -10,6 +10,7 @@ scheduler-k3s:autoscaling-auth:report [--format stdout|json] [--i scheduler-k3s:cluster-add [ssh://user@host:port] # Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list # Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id] # Removes client node to a Dokku-managed cluster +scheduler-k3s:ensure-charts # Ensures the k3s charts are installed scheduler-k3s:initialize # Initializes a cluster scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination scheduler-k3s:report [] [] # Displays a scheduler-k3s report for one or more apps diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index e369f2b3d..449103c2c 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,4 +1,4 @@ -SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall +SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index 719346d45..8f007cdf1 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -21,6 +21,7 @@ import ( "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/kube" + "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/storage/driver" ) @@ -58,9 +59,12 @@ type ChartInput struct { } type Release struct { - Name string - Namespace string - Version int + AppVersion string + Name string + Namespace string + Revision int + Status release.Status + Version string } type HelmAgent struct { @@ -269,24 +273,61 @@ func (h *HelmAgent) InstallChart(ctx context.Context, input ChartInput) error { return nil } -func (h *HelmAgent) ListRevisions(ctx context.Context, releaseName string) ([]Release, error) { - client := action.NewHistory(h.Configuration) - response, err := client.Run(releaseName) +func (h *HelmAgent) InstalledRevision(releaseName string) (Release, error) { + revisions, err := h.ListRevisions(ListRevisionsInput{ + ReleaseName: releaseName, + Max: 1, + }) if err != nil { - return nil, fmt.Errorf("Error getting revisions: %w", err) + return Release{}, err + } + + if len(revisions) == 0 { + return Release{}, nil + } + + return revisions[0], nil +} + +type ListRevisionsInput struct { + ReleaseName string + Max int +} + +func (h *HelmAgent) ListRevisions(input ListRevisionsInput) ([]Release, error) { + client := action.NewHistory(h.Configuration) + if input.Max > 0 { + client.Max = input.Max } releases := []Release{} + response, err := client.Run(input.ReleaseName) + if err != nil { + if errors.Is(err, driver.ErrReleaseNotFound) { + return releases, nil + } + + return nil, fmt.Errorf("Error getting revisions: %w", err) + } + for _, release := range response { + appVersion := "MISSING" + if release.Chart != nil && release.Chart.Metadata != nil { + appVersion = release.Chart.AppVersion() + } + releases = append(releases, Release{ - Name: release.Name, - Namespace: release.Namespace, - Version: release.Version, + AppVersion: appVersion, + Name: release.Name, + Namespace: release.Namespace, + Revision: release.Version, + Status: release.Info.Status, + Version: release.Chart.Metadata.Version, }) } sort.Slice(releases, func(i, j int) bool { - return releases[i].Version < releases[j].Version + return releases[i].Revision < releases[j].Revision }) return releases, nil diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index 702c3ba96..a87f59ad8 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -22,6 +22,7 @@ Additional commands:` scheduler-k3s:cluster-add [--insecure-allow-unknown-hosts] [--server-ip SERVER_IP] [--taint-scheduling] , Adds a server node to a Dokku-managed cluster scheduler-k3s:cluster-list [--format json|stdout], Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster-remove [node-id], Removes client node to a Dokku-managed cluster + scheduler-k3s:ensure-charts, Ensures the k3s charts are installed scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination scheduler-k3s:report [] [], Displays a scheduler-k3s report for one or more apps diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 883dc4a56..83c9e0db3 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -73,6 +73,10 @@ func main() { args.Parse(os.Args[2:]) nodeName := args.Arg(0) err = scheduler_k3s.CommandClusterRemove(nodeName) + case "ensure-charts": + args := flag.NewFlagSet("scheduler-k3s:ensure-charts", flag.ExitOnError) + args.Parse(os.Args[2:]) + err = scheduler_k3s.CommandEnsureCharts() case "initialize": args := flag.NewFlagSet("scheduler-k3s:initialize", flag.ExitOnError) taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index ac22af732..5691209e3 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -798,6 +798,96 @@ func CommandClusterRemove(nodeName string) error { return nil } +// CommandEnsureCharts ensures that the required helm charts are installed +func CommandEnsureCharts() error { + ctx, cancel := context.WithCancel(context.Background()) + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGQUIT, + syscall.SIGTERM) + go func() { + <-signals + cancel() + }() + + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Unable to create kubernetes client: %w", err) + } + + ingressClass := getGlobalIngressClass() + + namespacedHelmAgents := map[string]*HelmAgent{} + for _, chart := range HelmCharts { + _, ok := namespacedHelmAgents[chart.Namespace] + if !ok { + helmAgent, err := NewHelmAgent(chart.Namespace, DeployLogPrinter) + if err != nil { + common.LogWarn(fmt.Sprintf("Unable to create helm agent: %s", err.Error())) + return err + } + namespacedHelmAgents[chart.Namespace] = helmAgent + } + } + + common.LogInfo2Quiet("Installing helm charts") + err = installHelmCharts(ctx, clientset, func(chart HelmChart) bool { + common.LogInfo1(fmt.Sprintf("Processing chart %s@%s", chart.ReleaseName, chart.Version)) + if chart.ChartPath == "traefik" && ingressClass == "nginx" { + common.LogVerbose("Skipping chart due to ingress-class mismatch") + return false + } + + if chart.ChartPath == "ingress-nginx" && ingressClass == "traefik" { + common.LogVerbose("Skipping chart due to ingress-class mismatch") + return false + } + + if chart.ReleaseName == "keda-add-ons-http" { + common.LogVerbose("Forcing install of keda-add-ons-http") + return true + } + + helmAgent := namespacedHelmAgents[chart.Namespace] + latestRevision, err := helmAgent.InstalledRevision(chart.ReleaseName) + if err != nil { + common.LogWarn(fmt.Sprintf("Unable to get installed revision: %s", err)) + return false + } + + if latestRevision.Name == "" { + common.LogVerbose("Installing missing chart") + return true + } + + if latestRevision.Version != chart.Version { + common.LogVerbose(fmt.Sprintf("Installing chart due to version mismatch: %s != %s", latestRevision.AppVersion, chart.Version)) + return false + } + + common.LogVerbose("Skipping chart: already installed") + return false + }) + if err != nil { + return fmt.Errorf("Unable to install helm charts: %w", err) + } + + for _, manifest := range KubernetesManifests { + common.LogInfo2Quiet(fmt.Sprintf("Installing %s@%s", manifest.Name, manifest.Version)) + err = clientset.ApplyKubernetesManifest(ctx, ApplyKubernetesManifestInput{ + Manifest: manifest.Path, + }) + if err != nil { + return fmt.Errorf("Unable to apply kubernetes manifest: %w", err) + } + } + + common.LogInfo2Quiet("Done") + + return nil +} + // CommandLabelsSet set or clear a scheduler-k3s label for an app func CommandLabelsSet(appName string, processType string, resourceType string, key string, value string) error { if resourceType == "" {