github/dokku
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor: move crontab writing code to scheduler-docker-local plugin

Browse Source 
The implementation was very heavily docker-local specific and thus didn't make a ton of sense to keep in the cron plugin.

This also re-introduces changes from #5908 to parallelize the crontab generation (which is also specific to the docker-local scheduler) that were lost during a rebase.

The impetus for this change was implementing a `cron:run` command. The logic for actually running the command isn't scheduler specific - merely invoking `dokku run` with the appropriate arguments - but all the logic around how the commands were retrieved were, so the change was made pre-emptively.
This commit is contained in:
Jose Diaz-Gonzalez
2023-07-02 02:43:55 -04:00
parent eb6f85f083
commit afc318526c
16 changed files with 329 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
plugins/cron/Makefile
View File

@@ -1,6 +1,5 @@
SUBCOMMANDS = subcommands/list subcommands/report
TRIGGERS = triggers/cron-write triggers/install triggers/post-delete triggers/post-deploy triggers/report
BUILD = commands subcommands triggers
BUILD = commands subcommands
PLUGIN_NAME = cron
include ../../common.mk

78
plugins/cron/cron.go Normal file
View File

@@ -0,0 +1,78 @@
package cron
import (
"encoding/json"
"fmt"
"io/ioutil"
"strings"
appjson "github.com/dokku/dokku/plugins/app-json"
"github.com/dokku/dokku/plugins/common"
"github.com/multiformats/go-base36"
cronparser "github.com/robfig/cron/v3"
)
type TemplateCommand struct {
ID string
App string
Command string
Schedule string
AltCommand string
LogFile string
}
func (t TemplateCommand) CronCommand() string {
if t.AltCommand != "" {
if t.LogFile != "" {
return fmt.Sprintf("%s &>> %s", t.AltCommand, t.LogFile)
}
return t.AltCommand
}
return fmt.Sprintf("dokku run --cron-id %s %s %s", t.ID, t.App, t.Command)
}
func FetchCronEntries(appName string) ([]TemplateCommand, error) {
commands := []TemplateCommand{}
appjsonPath := appjson.GetAppjsonPath(appName)
if !common.FileExists(appjsonPath) {
return commands, nil
}
b, err := ioutil.ReadFile(appjsonPath)
if err != nil {
return commands, fmt.Errorf("Cannot read app.json file for %s: %v", appName, err)
}
if strings.TrimSpace(string(b)) == "" {
return commands, nil
}
var appJSON appjson.AppJSON
if err = json.Unmarshal(b, &appJSON); err != nil {
return commands, fmt.Errorf("Cannot parse app.json for %s: %v", appName, err)
}
for _, c := range appJSON.Cron {
parser := cronparser.NewParser(cronparser.Minute | cronparser.Hour | cronparser.Dom | cronparser.Month | cronparser.Dow | cronparser.Descriptor)
_, err := parser.Parse(c.Schedule)
if err != nil {
return commands, err
}
commands = append(commands, TemplateCommand{
App: appName,
Command: c.Command,
Schedule: c.Schedule,
ID: GenerateCommandID(appName, c),
})
}
return commands, nil
}
// GenerateCommandID creates a unique ID for a given app/command/schedule combination
func GenerateCommandID(appName string, c appjson.CronCommand) string {
return base36.EncodeToStringLc([]byte(appName + "===" + c.Command + "===" + c.Schedule))
}

193
plugins/cron/functions.go
View File

@@ -1,193 +0,0 @@
package cron
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"text/template"
appjson "github.com/dokku/dokku/plugins/app-json"
"github.com/dokku/dokku/plugins/common"
base36 "github.com/multiformats/go-base36"
cronparser "github.com/robfig/cron/v3"
)
type templateCommand struct {
ID string
App string
Command string
Schedule string
AltCommand string
LogFile string
}
func (t templateCommand) CronCommand() string {
if t.AltCommand != "" {
if t.LogFile != "" {
return fmt.Sprintf("%s &>> %s", t.AltCommand, t.LogFile)
}
return t.AltCommand
}
return fmt.Sprintf("dokku run --cron-id %s %s %s", t.ID, t.App, t.Command)
}
func fetchCronEntries(appName string) ([]templateCommand, error) {
commands := []templateCommand{}
scheduler := common.GetAppScheduler(appName)
if scheduler != "docker-local" {
return commands, nil
}
appjsonPath := appjson.GetAppjsonPath(appName)
if !common.FileExists(appjsonPath) {
return commands, nil
}
b, err := ioutil.ReadFile(appjsonPath)
if err != nil {
return commands, fmt.Errorf("Cannot read app.json file for %s: %v", appName, err)
}
if strings.TrimSpace(string(b)) == "" {
return commands, nil
}
var appJSON appjson.AppJSON
if err = json.Unmarshal(b, &appJSON); err != nil {
return commands, fmt.Errorf("Cannot parse app.json for %s: %v", appName, err)
}
for _, c := range appJSON.Cron {
parser := cronparser.NewParser(cronparser.Minute | cronparser.Hour | cronparser.Dom | cronparser.Month | cronparser.Dow | cronparser.Descriptor)
_, err := parser.Parse(c.Schedule)
if err != nil {
return commands, err
}
commands = append(commands, templateCommand{
App: appName,
Command: c.Command,
Schedule: c.Schedule,
ID: generateCommandID(appName, c),
})
}
return commands, nil
}
func deleteCrontab() error {
command := common.NewShellCmd("sudo /usr/bin/crontab -l -u dokku")
command.ShowOutput = false
if !command.Execute() {
return nil
}
command = common.NewShellCmd("sudo /usr/bin/crontab -r -u dokku")
command.ShowOutput = false
out, err := command.CombinedOutput()
if err != nil {
return fmt.Errorf("Unable to remove schedule file: %v", string(out))
}
common.LogInfo1("Removed")
return nil
}
func writeCronEntries() error {
apps, _ := common.UnfilteredDokkuApps()
commands := []templateCommand{}
for _, appName := range apps {
scheduler := common.GetAppScheduler(appName)
if scheduler != "docker-local" {
continue
}
c, err := fetchCronEntries(appName)
if err != nil {
return err
}
commands = append(commands, c...)
}
b, _ := common.PlugnTriggerOutput("cron-entries", "docker-local")
for _, line := range strings.Split(strings.TrimSpace(string(b[:])), "\n") {
if strings.TrimSpace(line) == "" {
continue
}
parts := strings.Split(line, ";")
if len(parts) != 2 && len(parts) != 3 {
return fmt.Errorf("Invalid injected cron task: %v", line)
}
id := base36.EncodeToStringLc([]byte(strings.Join(parts, ";;;")))
command := templateCommand{
ID: id,
Schedule: parts[0],
AltCommand: parts[1],
}
if len(parts) == 3 {
command.LogFile = parts[2]
}
commands = append(commands, command)
}
if len(commands) == 0 {
return deleteCrontab()
}
data := map[string]interface{}{
"Commands": commands,
}
t, err := getCronTemplate()
if err != nil {
return err
}
tmpFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "WriteCronEntries"))
if err != nil {
return fmt.Errorf("Cannot create temporary schedule file: %v", err)
}
defer tmpFile.Close()
defer os.Remove(tmpFile.Name())
if err := t.Execute(tmpFile, data); err != nil {
return fmt.Errorf("Unable to template out schedule file: %v", err)
}
command := common.NewShellCmd(fmt.Sprintf("sudo /usr/bin/crontab -u dokku %s", tmpFile.Name()))
command.ShowOutput = false
out, err := command.CombinedOutput()
if err != nil {
return fmt.Errorf("Unable to update schedule file: %s", string(out))
}
common.LogInfo1("Updated schedule file")
return nil
}
func getCronTemplate() (*template.Template, error) {
t := template.New("cron")
templatePath := filepath.Join(common.MustGetEnv("PLUGIN_ENABLED_PATH"), "cron", "templates", "cron.tmpl")
b, err := ioutil.ReadFile(templatePath)
if err != nil {
return t, fmt.Errorf("Cannot read template file: %v", err)
}
s := strings.TrimSpace(string(b))
return t.Parse(s)
}
func generateCommandID(appName string, c appjson.CronCommand) string {
return base36.EncodeToStringLc([]byte(appName + "===" + c.Command + "===" + c.Schedule))
}

2
plugins/cron/report.go
View File

@@ -28,6 +28,6 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
}
func reportTasks(appName string) string {
c, _ := fetchCronEntries(appName)
c, _ := FetchCronEntries(appName)
return strconv.Itoa(len(c))
}

2
plugins/cron/subcommands.go
View File

@@ -13,7 +13,7 @@ func CommandList(appName string) error {
return err
}
entries, err := fetchCronEntries(appName)
entries, err := FetchCronEntries(appName)
if err != nil {
return err
}

26
plugins/cron/triggers.go
View File

@@ -1,26 +0,0 @@
package cron
import (
"github.com/dokku/dokku/plugins/common"
)
// TriggerInstall installs a sudoers file so we can execute crontab via sudo
func TriggerInstall() error {
lines := []string{"%dokku ALL=(ALL) NOPASSWD:/usr/bin/crontab"}
return common.WriteSliceToFile("/etc/sudoers.d/dokku-cron", lines)
}
// TriggerPostDelete updates the cron entries for all apps
func TriggerPostDelete() error {
return writeCronEntries()
}
// TriggerPostDeploy updates the cron entries for all apps
func TriggerPostDeploy() error {
return writeCronEntries()
}
// TriggerCronWrite force updates the cron file for all apps
func TriggerCronWrite() error {
return writeCronEntries()
}

3
plugins/scheduler-docker-local/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
/triggers/*
/triggers
/cron-*

5
plugins/scheduler-docker-local/Makefile Normal file
View File

@@ -0,0 +1,5 @@
TRIGGERS = triggers/cron-write
BUILD = triggers
PLUGIN_NAME = scheduler-docker-local
include ../../common.mk

163
plugins/scheduler-docker-local/functions.go Normal file
View File

@@ -0,0 +1,163 @@
package schedulerdockerlocal
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"text/template"
"github.com/dokku/dokku/plugins/common"
"github.com/dokku/dokku/plugins/cron"
"golang.org/x/sync/errgroup"
base36 "github.com/multiformats/go-base36"
)
func deleteCrontab() error {
command := common.NewShellCmd("sudo /usr/bin/crontab -l -u dokku")
command.ShowOutput = false
if !command.Execute() {
return nil
}
command = common.NewShellCmd("sudo /usr/bin/crontab -r -u dokku")
command.ShowOutput = false
out, err := command.CombinedOutput()
if err != nil {
return fmt.Errorf("Unable to remove schedule file: %v", string(out))
}
common.LogInfo1("Removed")
return nil
}
func generateCronEntries() ([]cron.TemplateCommand, error) {
apps, _ := common.UnfilteredDokkuApps()
g := new(errgroup.Group)
results := make(chan []cron.TemplateCommand, len(apps)+1)
for _, appName := range apps {
appName := appName
g.Go(func() error {
scheduler := common.GetAppScheduler(appName)
if scheduler != "docker-local" {
results <- []cron.TemplateCommand{}
return nil
}
c, err := cron.FetchCronEntries(appName)
if err != nil {
results <- []cron.TemplateCommand{}
return err
}
results <- c
return nil
})
}
g.Go(func() error {
commands := []cron.TemplateCommand{}
b, _ := common.PlugnTriggerOutput("cron-entries", "docker-local")
for _, line := range strings.Split(strings.TrimSpace(string(b[:])), "\n") {
if strings.TrimSpace(line) == "" {
results <- []cron.TemplateCommand{}
return nil
}
parts := strings.Split(line, ";")
if len(parts) != 2 && len(parts) != 3 {
results <- []cron.TemplateCommand{}
return fmt.Errorf("Invalid injected cron task: %v", line)
}
id := base36.EncodeToStringLc([]byte(strings.Join(parts, ";;;")))
command := cron.TemplateCommand{
ID: id,
Schedule: parts[0],
AltCommand: parts[1],
}
if len(parts) == 3 {
command.LogFile = parts[2]
}
commands = append(commands, command)
}
results <- commands
return nil
})
err := g.Wait()
close(results)
commands := []cron.TemplateCommand{}
if err != nil {
return commands, err
}
for result := range results {
c := result
if len(c) > 0 {
commands = append(commands, c...)
}
}
return commands, nil
}
func writeCronEntries() error {
commands, err := generateCronEntries()
if err != nil {
return err
}
if len(commands) == 0 {
return deleteCrontab()
}
data := map[string]interface{}{
"Commands": commands,
}
t, err := getCronTemplate()
if err != nil {
return err
}
tmpFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("dokku-%s-%s", common.MustGetEnv("DOKKU_PID"), "WriteCronEntries"))
if err != nil {
return fmt.Errorf("Cannot create temporary schedule file: %v", err)
}
defer tmpFile.Close()
defer os.Remove(tmpFile.Name())
if err := t.Execute(tmpFile, data); err != nil {
return fmt.Errorf("Unable to template out schedule file: %v", err)
}
command := common.NewShellCmd(fmt.Sprintf("sudo /usr/bin/crontab -u dokku %s", tmpFile.Name()))
command.ShowOutput = false
out, err := command.CombinedOutput()
if err != nil {
return fmt.Errorf("Unable to update schedule file: %s", string(out))
}
common.LogInfo1("Updated schedule file")
return nil
}
func getCronTemplate() (*template.Template, error) {
t := template.New("cron")
templatePath := filepath.Join(common.MustGetEnv("PLUGIN_ENABLED_PATH"), "cron", "templates", "cron.tmpl")
b, err := ioutil.ReadFile(templatePath)
if err != nil {
return t, fmt.Errorf("Cannot read template file: %v", err)
}
s := strings.TrimSpace(string(b))
return t.Parse(s)
}

27
plugins/scheduler-docker-local/go.mod Normal file
View File

@@ -0,0 +1,27 @@
module github.com/dokku/dokku/plugins/scheduler-docker-local
go 1.19
require (
github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000
github.com/dokku/dokku/plugins/cron v0.0.0-00010101000000-000000000000
github.com/multiformats/go-base36 v0.2.0
golang.org/x/sync v0.3.0
)
require (
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 // indirect
github.com/dokku/dokku/plugins/app-json v0.0.0-00010101000000-000000000000 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/otiai10/copy v1.12.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/ryanuber/columnize v2.1.2+incompatible // indirect
golang.org/x/sys v0.8.0 // indirect
)
replace github.com/dokku/dokku/plugins/app-json => ../app-json
replace github.com/dokku/dokku/plugins/common => ../common
replace github.com/dokku/dokku/plugins/cron => ../cron

24
plugins/scheduler-docker-local/go.sum Normal file
View File

@@ -0,0 +1,24 @@
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF25UfsK2X7Ph/zCAk=
github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

3
plugins/scheduler-docker-local/install
View File

@@ -14,6 +14,9 @@ trigger-scheduler-docker-local-install() {
fn-plugin-property-setup "scheduler-docker-local"
echo "%dokku ALL=(ALL) NOPASSWD:/usr/bin/crontab" >"/etc/sudoers.d/dokku-cron"
chmod "0440" "/etc/sudoers.d/dokku-cron"
DOKKU_PATH="$(command -v dokku)"
if [[ $(systemctl 2>/dev/null) =~ -\.mount ]]; then

2
plugins/scheduler-docker-local/post-delete
View File

@@ -14,6 +14,8 @@ trigger-scheduler-docker-local-post-delete() {
# remove all application containers
"$DOCKER_BIN" container ls --filter "label=com.dokku.app-name=${APP}" -q | xargs -n1 -I {} "$DOCKER_BIN" container rm --force {} &>/dev/null || true
plugn trigger cron-write
}
trigger-scheduler-docker-local-post-delete "$@"

13
plugins/scheduler-docker-local/post-deploy Executable file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/env bash
set -eo pipefail
[[ $DOKKU_TRACE ]] && set -x
trigger-scheduler-docker-local-post-deploy() {
declare desc="scheduler-docker-local post-deploy plugin trigger"
declare trigger="post-deploy"
declare APP="$1"
plugn trigger cron-write
}
trigger-scheduler-docker-local-post-deploy "$@"

13
plugins/cron/src/triggers/triggers.go → plugins/scheduler-docker-local/src/triggers/triggers.go
View File

@@ -7,7 +7,7 @@ import (
"strings"
"github.com/dokku/dokku/plugins/common"
cron "github.com/dokku/dokku/plugins/cron"
schedulerdockerlocal "github.com/dokku/dokku/plugins/scheduler-docker-local"
)
// main entrypoint to all triggers
@@ -19,16 +19,7 @@ func main() {
var err error
switch trigger {
case "cron-write":
err = cron.TriggerCronWrite()
case "install":
err = cron.TriggerInstall()
case "post-delete":
err = cron.TriggerPostDelete()
case "post-deploy":
err = cron.TriggerPostDeploy()
case "report":
appName := flag.Arg(0)
err = cron.ReportSingleApp(appName, "", "")
err = schedulerdockerlocal.TriggerCronWrite()
default:
err = fmt.Errorf("Invalid plugin trigger call: %s", trigger)
}

6
plugins/scheduler-docker-local/triggers.go Normal file
View File

@@ -0,0 +1,6 @@
package schedulerdockerlocal
// TriggerCronWrite force updates the cron file for all apps
func TriggerCronWrite() error {
return writeCronEntries()
}