├── VERSION ├── .gitignore ├── lib ├── templater_test.go ├── config.go ├── templater.go ├── execution.go ├── graph.go ├── ui.go ├── graph_test.go ├── executor_test.go ├── types.go ├── names.go └── executor.go ├── assets └── hello-world.graph.png ├── .travis.yml ├── Dockerfile ├── examples ├── hello-world.yaml └── environment.yaml ├── .goreleaser.yml ├── Makefile ├── .editorconfig ├── go.mod ├── main.go ├── ci └── ci.yml ├── README.md └── go.sum /VERSION: -------------------------------------------------------------------------------- 1 | 0.0.2-rc.8 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | graph.png 2 | dist/ 3 | -------------------------------------------------------------------------------- /lib/templater_test.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | // TODO 4 | -------------------------------------------------------------------------------- /assets/hello-world.graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cirocosta/cr/HEAD/assets/hello-world.graph.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | language: 'go' 4 | 5 | go: 6 | - '1.9' 7 | 8 | script: 9 | - 'make test' 10 | 11 | notifications: 12 | email: false 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1 2 | 3 | RUN mkdir /src 4 | ADD . /src 5 | WORKDIR /src 6 | 7 | RUN go install -ldflags "-X github.com/cirocosta/cr.version=$(cat ./VERSION)" -v 8 | -------------------------------------------------------------------------------- /examples/hello-world.yaml: -------------------------------------------------------------------------------- 1 | Jobs: 2 | - Id: 'SayFoo' 3 | Run: 'echo foo' 4 | 5 | - Id: 'SayBaz' 6 | Run: 'echo baz' 7 | DependsOn: [ 'SayFoo' ] 8 | 9 | - Id: 'SayCaz' 10 | Run: 'echo caz' 11 | DependsOn: [ 'SayFoo' ] 12 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | build: 2 | flags: '-tags netgo' 3 | ldflags: '-X main.version={{ .Version }}' 4 | env: 5 | - 'CGO_ENABLED=0' 6 | goos: 7 | - 'linux' 8 | - 'darwin' 9 | goarch: 10 | - 'amd64' 11 | archive: 12 | format: 'tar.gz' 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell cat ./VERSION) 2 | 3 | install: 4 | go install -v 5 | 6 | fmt: 7 | go fmt ./... 8 | 9 | test: 10 | cd ./lib && go test -v 11 | 12 | release: 13 | git tag -a $(VERSION) -m "Release" || true 14 | git push origin $(VERSION) 15 | goreleaser --rm-dist 16 | 17 | .PHONY: fmt install test release 18 | 19 | -------------------------------------------------------------------------------- /examples/environment.yaml: -------------------------------------------------------------------------------- 1 | Env: 2 | WORD: 'BAZ' 3 | 4 | Jobs: 5 | - Id: 'SayFoo' 6 | Run: 'echo $WORD' 7 | Env: 8 | WORD: 'foo' 9 | 10 | - Id: 'SayBaz' 11 | Run: 'echo $WORD' 12 | DependsOn: [ 'SayFoo' ] 13 | 14 | - Id: 'SayCaz' 15 | Run: 'echo $SAY' 16 | Env: 17 | SAY: 'CAZ' 18 | DependsOn: [ 'SayFoo' ] 19 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = LF 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | indent_style = space 9 | indent_size = 8 10 | 11 | [Makefile] 12 | indent_size = 4 13 | indent_style = tab 14 | 15 | [*.md] 16 | trim_trailing_whitespace = false 17 | 18 | [*.go] 19 | indent_style = tab 20 | 21 | [*.yml] 22 | indent_style = space 23 | indent_size = 2 24 | 25 | [*.sh] 26 | indent_style = space 27 | indent_size = 2 28 | 29 | -------------------------------------------------------------------------------- /lib/config.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/pkg/errors" 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | func ConfigFromFile(file string) (config Config, err error) { 12 | finfo, err := os.Open(file) 13 | if err != nil { 14 | if os.IsNotExist(err) { 15 | err = errors.Wrapf(err, 16 | "configuration file %s not found", 17 | file) 18 | return 19 | } 20 | 21 | err = errors.Wrapf(err, 22 | "unexpected error looking for config file %s", 23 | file) 24 | return 25 | } 26 | 27 | configContent, err := ioutil.ReadAll(finfo) 28 | if err != nil { 29 | err = errors.Wrapf(err, 30 | "couldn't properly read config file %s", 31 | file) 32 | return 33 | } 34 | 35 | err = yaml.Unmarshal(configContent, &config) 36 | if err != nil { 37 | err = errors.Wrapf(err, 38 | "couldn't properly parse yaml config file %s", 39 | file) 40 | return 41 | } 42 | 43 | return 44 | } 45 | -------------------------------------------------------------------------------- /lib/templater.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "text/template" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | var ( 12 | FuncMap = template.FuncMap{ 13 | "env": os.Getenv, 14 | } 15 | ) 16 | 17 | // TemplateField takes a field string and a state. 18 | // With that it applies the state in the template and 19 | // generates a response. 20 | func TemplateField(field string, state *RenderState) (res string, err error) { 21 | var ( 22 | tmpl *template.Template 23 | output bytes.Buffer 24 | ) 25 | 26 | tmpl, err = template. 27 | New("tmpl"). 28 | Funcs(FuncMap). 29 | Parse(field) 30 | if err != nil { 31 | err = errors.Wrapf(err, 32 | "failed to instantiate template for record '%s'", 33 | field) 34 | return 35 | } 36 | 37 | err = tmpl.Execute(&output, state) 38 | if err != nil { 39 | err = errors.Wrapf(err, "failed to execute template") 40 | return 41 | } 42 | 43 | res = output.String() 44 | 45 | return 46 | } 47 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module cr 2 | 3 | require ( 4 | github.com/alexflint/go-arg v0.0.0-20170330211029-cef6506c97e5 5 | github.com/alexflint/go-scalar v0.0.0-20170216015739-45e5d6cd8605 // indirect 6 | github.com/cirocosta/cr v0.0.0-20171218233017-dc8c1f40647b 7 | github.com/davecgh/go-spew v1.1.1 // indirect 8 | github.com/fatih/color v0.0.0-20170926111411-5df930a27be2 9 | github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect 10 | github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 // indirect 11 | github.com/hashicorp/logutils v1.0.0 // indirect 12 | github.com/hashicorp/terraform v0.0.0-20171212233002-681b2e75875e 13 | github.com/kr/pretty v0.1.0 // indirect 14 | github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597 // indirect 15 | github.com/mattn/go-isatty v0.0.0-20170307163044-57fdcb988a5c // indirect 16 | github.com/pkg/errors v0.8.0 17 | github.com/pmezard/go-difflib v1.0.0 // indirect 18 | github.com/rs/zerolog v1.3.0 19 | github.com/stretchr/testify v1.1.4 20 | golang.org/x/sys v0.0.0-20170213225739-e24f485414ae // indirect 21 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 22 | gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab 23 | ) 24 | -------------------------------------------------------------------------------- /lib/execution.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/exec" 7 | "syscall" 8 | "time" 9 | 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | const ( 14 | defaultFailedExitCode int = 1 15 | ) 16 | 17 | // init initializes the Execution parameters that rely on 18 | // initialization. It also sets default parameters 19 | // that might not be set. 20 | func (e *Execution) init(ctx context.Context) (err error) { 21 | if len(e.Argv) == 0 { 22 | err = errors.Errorf("Argv must have at least one element") 23 | return 24 | } 25 | 26 | allEnv := os.Environ() 27 | for k, v := range e.Env { 28 | allEnv = append(allEnv, k+"="+v) 29 | } 30 | 31 | e.cmd = exec.CommandContext(ctx, e.Argv[0], e.Argv[1:]...) 32 | e.cmd.Stdout = e.Stdout 33 | e.cmd.Stderr = e.Stderr 34 | e.cmd.Dir = e.Directory 35 | e.cmd.Env = allEnv 36 | 37 | return 38 | } 39 | 40 | // Run is a blocking method that executes the desired command 41 | // tying it to a context which, when cancelled, kills the process. 42 | func (e *Execution) Run(ctx context.Context) (err error) { 43 | err = e.init(ctx) 44 | if err != nil { 45 | err = errors.Wrapf(err, "Couldn't initialize execution") 46 | return 47 | } 48 | 49 | e.StartTime = time.Now() 50 | err = e.cmd.Run() 51 | e.EndTime = time.Now() 52 | if err != nil { 53 | if exitError, ok := err.(*exec.ExitError); ok { 54 | ws := exitError.Sys().(syscall.WaitStatus) 55 | e.ExitCode = ws.ExitStatus() 56 | } else { 57 | e.ExitCode = defaultFailedExitCode 58 | } 59 | } else { 60 | ws := e.cmd.ProcessState.Sys().(syscall.WaitStatus) 61 | e.ExitCode = ws.ExitStatus() 62 | } 63 | 64 | return 65 | } 66 | -------------------------------------------------------------------------------- /lib/graph.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "github.com/hashicorp/terraform/dag" 5 | "github.com/pkg/errors" 6 | ) 7 | 8 | // BuildDependencyGraph creates a DAG with 9 | // a dumb root aiming at providing the 10 | // biggest possible parallelism to a series 11 | // of job builds. 12 | func BuildDependencyGraph(jobs []*Job) (g dag.AcyclicGraph, err error) { 13 | var ( 14 | rootJob = &Job{ 15 | Id: "_root", 16 | } 17 | jobsMap = map[string]*Job{ 18 | "_root": rootJob, 19 | } 20 | job *Job 21 | dep string 22 | present bool 23 | ) 24 | 25 | if jobs == nil { 26 | err = errors.Errorf("jobs can't be nil") 27 | return 28 | } 29 | 30 | g.Add(rootJob) 31 | 32 | for _, job = range jobs { 33 | if job.Id == "" { 34 | err = errors.Errorf( 35 | "job must have an Id - %+v", 36 | job) 37 | return 38 | } 39 | 40 | g.Add(job) 41 | 42 | _, present = jobsMap[job.Id] 43 | if present { 44 | err = errors.Errorf( 45 | "can't have two jobs with the same id - %s", 46 | job.Id) 47 | return 48 | } 49 | 50 | jobsMap[job.Id] = job 51 | } 52 | 53 | for _, job = range jobs { 54 | if len(job.DependsOn) == 0 { 55 | g.Connect(dag.BasicEdge(rootJob, job)) 56 | continue 57 | } 58 | 59 | for _, dep = range job.DependsOn { 60 | depJob, present := jobsMap[dep] 61 | if !present { 62 | err = errors.Errorf( 63 | "job %s has a dependency %s "+ 64 | "that does not exist", 65 | job.Id, dep) 66 | return 67 | } 68 | 69 | g.Connect(dag.BasicEdge(depJob, job)) 70 | } 71 | } 72 | 73 | err = g.Validate() 74 | if err != nil { 75 | err = errors.Wrapf(err, "jobs graph is invalid") 76 | return 77 | } 78 | 79 | return 80 | } 81 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "math/rand" 9 | "os" 10 | "time" 11 | 12 | "github.com/alexflint/go-arg" 13 | "github.com/cirocosta/cr/lib" 14 | "github.com/rs/zerolog" 15 | ) 16 | 17 | var version string = "dev" 18 | 19 | type cliArgs struct { 20 | lib.Runtime 21 | } 22 | 23 | func (r cliArgs) Version() string { 24 | return "cr - the concurrent runner - version=" + version 25 | } 26 | 27 | var ( 28 | args = &cliArgs{ 29 | lib.Runtime{ 30 | File: "./.cr.yml", 31 | LogsDirectory: "/tmp", 32 | Stdout: false, 33 | Graph: false, 34 | }, 35 | } 36 | logger = zerolog.New(os.Stdout). 37 | With(). 38 | Str("from", "main"). 39 | Logger() 40 | ui = lib.NewUi() 41 | ) 42 | 43 | func must(err error) { 44 | if err == nil { 45 | return 46 | } 47 | 48 | logger.Fatal(). 49 | Err(err). 50 | Msg("main execution failed") 51 | } 52 | 53 | func main() { 54 | arg.MustParse(args) 55 | 56 | rand.Seed(time.Now().UnixNano()) 57 | log.SetOutput(ioutil.Discard) 58 | 59 | cfg, err := lib.ConfigFromFile(args.File) 60 | must(err) 61 | 62 | cfg.OnJobStatusChange = func(a *lib.Activity) { 63 | ui.WriteActivity(a) 64 | } 65 | 66 | if cfg.Runtime.LogsDirectory == "" { 67 | cfg.Runtime.LogsDirectory = args.LogsDirectory 68 | } 69 | 70 | if args.Stdout { 71 | cfg.Runtime.Stdout = true 72 | } 73 | 74 | executor, err := lib.New(&cfg) 75 | must(err) 76 | 77 | if args.Graph { 78 | fmt.Println(executor.GetDotGraph()) 79 | os.Exit(0) 80 | } 81 | 82 | fmt.Printf(` 83 | Starting execution. 84 | 85 | Logs directory: %s 86 | `+"\n", cfg.Runtime.LogsDirectory) 87 | 88 | err = executor.Execute(context.Background()) 89 | must(err) 90 | } 91 | -------------------------------------------------------------------------------- /lib/ui.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | "text/tabwriter" 7 | "time" 8 | 9 | "github.com/fatih/color" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | type ActivityType int 14 | 15 | const ( 16 | ActivityUnknown ActivityType = iota 17 | ActivityStarted 18 | ActivityErrored 19 | ActivitySuccess 20 | ActivityAborted 21 | ) 22 | 23 | type Activity struct { 24 | Type ActivityType 25 | Time time.Time 26 | Job *Job 27 | } 28 | 29 | var ( 30 | ActivityMapping = map[ActivityType]string{ 31 | ActivityAborted: "ABORTED", 32 | ActivityStarted: "STARTED", 33 | ActivityErrored: "ERRORED", 34 | ActivitySuccess: "SUCCESS", 35 | ActivityUnknown: "UNKNOWN", 36 | } 37 | WriterMapping = map[ActivityType]*color.Color{ 38 | ActivityAborted: color.New(color.FgYellow), 39 | ActivityStarted: color.New(color.FgBlue), 40 | ActivityErrored: color.New(color.FgRed), 41 | ActivitySuccess: color.New(color.FgGreen), 42 | ActivityUnknown: color.New(color.FgCyan), 43 | } 44 | ) 45 | 46 | type Ui struct { 47 | writer *tabwriter.Writer 48 | sync.Mutex 49 | } 50 | 51 | func NewUi() (u Ui) { 52 | u.writer = new(tabwriter.Writer) 53 | u.writer.Init(os.Stdout, 10, 8, 2, '\t', 0) 54 | 55 | return 56 | } 57 | 58 | func (u *Ui) WriteActivity(a *Activity) (err error) { 59 | u.Lock() 60 | defer u.Unlock() 61 | 62 | switch a.Type { 63 | case ActivityStarted: 64 | WriterMapping[a.Type]. 65 | Fprintf(u.writer, "%s\tstatus=%s\tstart=%s\n", 66 | a.Job.Id, 67 | ActivityMapping[a.Type], 68 | time.Now().Format("15:04:05")) 69 | case ActivityErrored, ActivitySuccess, ActivityAborted: 70 | WriterMapping[a.Type]. 71 | Fprintf(u.writer, "%s\tstatus=%s\tstart=%s\telapsed=%s\n", 72 | a.Job.Id, 73 | ActivityMapping[a.Type], 74 | a.Job.StartTime.Format("15:04:05"), 75 | a.Job.EndTime.Sub(*a.Job.StartTime).String()) 76 | default: 77 | err = errors.Errorf( 78 | "unknown activity type %+v", a) 79 | return 80 | } 81 | 82 | u.writer.Flush() 83 | return 84 | } 85 | -------------------------------------------------------------------------------- /lib/graph_test.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/hashicorp/terraform/dag" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestBuildDependencyGraph(t *testing.T) { 13 | var testCases = []struct { 14 | desc string 15 | jobs []*Job 16 | expected string 17 | shouldFail bool 18 | }{ 19 | { 20 | desc: "nil should fail", 21 | shouldFail: true, 22 | }, 23 | { 24 | desc: "single job", 25 | jobs: []*Job{ 26 | { 27 | Id: "job1", 28 | }, 29 | }, 30 | expected: ` 31 | _root 32 | job1 33 | job1`, 34 | }, 35 | { 36 | desc: "two jobs with no deps", 37 | jobs: []*Job{ 38 | { 39 | Id: "job1", 40 | }, 41 | { 42 | Id: "job2", 43 | }, 44 | }, 45 | expected: ` 46 | _root 47 | job1 48 | job2 49 | job1 50 | job2`, 51 | }, 52 | { 53 | desc: "two jobs with single dependency", 54 | jobs: []*Job{ 55 | { 56 | Id: "job1", 57 | }, 58 | { 59 | Id: "job2", 60 | DependsOn: []string{ 61 | "job1", 62 | }, 63 | }, 64 | }, 65 | expected: ` 66 | _root 67 | job1 68 | job1 69 | job2 70 | job2`, 71 | }, 72 | { 73 | desc: "thre jobs with two jobs depending in one", 74 | jobs: []*Job{ 75 | { 76 | Id: "job1", 77 | }, 78 | { 79 | Id: "job2", 80 | DependsOn: []string{ 81 | "job1", 82 | }, 83 | }, 84 | { 85 | Id: "job3", 86 | DependsOn: []string{ 87 | "job1", 88 | }, 89 | }, 90 | }, 91 | expected: ` 92 | _root 93 | job1 94 | job1 95 | job2 96 | job3 97 | job2 98 | job3`, 99 | }, 100 | { 101 | desc: "three jobs with serial dependency", 102 | jobs: []*Job{ 103 | { 104 | Id: "job1", 105 | }, 106 | { 107 | Id: "job2", 108 | DependsOn: []string{ 109 | "job1", 110 | }, 111 | }, 112 | { 113 | Id: "job3", 114 | DependsOn: []string{ 115 | "job2", 116 | }, 117 | }, 118 | }, 119 | expected: ` 120 | _root 121 | job1 122 | job1 123 | job2 124 | job2 125 | job3 126 | job3`, 127 | }, 128 | { 129 | desc: "cyclic dependency", 130 | jobs: []*Job{ 131 | { 132 | Id: "job1", 133 | DependsOn: []string{ 134 | "job2", 135 | }, 136 | }, 137 | { 138 | Id: "job2", 139 | DependsOn: []string{ 140 | "job1", 141 | }, 142 | }, 143 | }, 144 | shouldFail: true, 145 | }, 146 | } 147 | 148 | var ( 149 | err error 150 | actual string 151 | expected string 152 | graph dag.AcyclicGraph 153 | ) 154 | 155 | // TODO add a root dummy job 156 | 157 | for _, tc := range testCases { 158 | t.Run(tc.desc, func(t *testing.T) { 159 | graph, err = BuildDependencyGraph(tc.jobs) 160 | if tc.shouldFail { 161 | require.Error(t, err) 162 | return 163 | } 164 | 165 | require.NoError(t, err) 166 | 167 | actual = strings.Trim(graph.String(), "\n") 168 | expected = strings.Trim(tc.expected, "\n") 169 | 170 | assert.Equal(t, expected, actual) 171 | }) 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /ci/ci.yml: -------------------------------------------------------------------------------- 1 | resource_types: 2 | - name: semver 3 | type: registry-image 4 | source: 5 | repository: concourse/semver-resource 6 | 7 | - name: github-release 8 | type: registry-image 9 | source: 10 | repository: concourse/github-release-resource 11 | 12 | resources: 13 | - name: repository 14 | type: git 15 | source: 16 | uri: https://github.com/cirocosta/cr 17 | ignore_paths: [ ./VERSION ] 18 | 19 | - name: builder 20 | type: registry-image 21 | source: 22 | repository: concourse/builder 23 | 24 | - name: golang 25 | type: registry-image 26 | source: 27 | repository: library/golang 28 | tag: '1' 29 | 30 | - name: version 31 | type: semver 32 | source: 33 | driver: git 34 | initial_version: 0.0.3 35 | uri: https://((github-token))@github.com/cirocosta/cr 36 | branch: master 37 | file: ./VERSION 38 | git_user: Ciro S. Costa 39 | 40 | - name: release 41 | type: github-release 42 | source: 43 | owner: cirocosta 44 | repository: cr 45 | access_token: ((github-token)) 46 | 47 | - name: docker-image 48 | type: registry-image 49 | source: 50 | repository: cirocosta/cr 51 | username: ((docker-user)) 52 | password: ((docker-password)) 53 | 54 | 55 | jobs: 56 | - name: test 57 | plan: 58 | - aggregate: 59 | - get: repository 60 | trigger: true 61 | - get: golang 62 | trigger: true 63 | - task: test 64 | image: golang 65 | config: 66 | platform: linux 67 | inputs: 68 | - name: repository 69 | path: . 70 | run: 71 | path: go 72 | args: [ 'test', './...' ] 73 | 74 | - name: release 75 | plan: 76 | - aggregate: 77 | - get: repository 78 | passed: [ 'test' ] 79 | trigger: true 80 | - get: golang 81 | passed: [ 'test' ] 82 | trigger: true 83 | - get: version 84 | - task: build 85 | image: golang 86 | config: 87 | platform: linux 88 | inputs: 89 | - name: repository 90 | path: . 91 | - name: version 92 | outputs: 93 | - name: artifacts 94 | run: 95 | path: /bin/bash 96 | args: 97 | - -c 98 | - -e 99 | - | 100 | go build -o ./artifacts/cr -v -ldflags "-X github.com/cirocosta/cr.version=$(cat version/version)" 101 | - put: release 102 | inputs: [ version, artifacts ] 103 | params: 104 | name: ./version/version 105 | tag: ./version/version 106 | globs: [ ./artifacts/cr ] 107 | - put: version 108 | params: 109 | pre: rc 110 | 111 | 112 | - name: image 113 | plan: 114 | - aggregate: 115 | - get: repository 116 | passed: [ 'test' ] 117 | trigger: true 118 | - get: builder 119 | trigger: true 120 | - get: version 121 | - task: build 122 | privileged: true 123 | image: builder 124 | config: 125 | platform: linux 126 | params: 127 | REPOSITORY: cirocosta/cr 128 | inputs: 129 | - name: repository 130 | path: . 131 | outputs: 132 | - name: image 133 | run: 134 | path: /bin/bash 135 | args: [ '-ce', 'TAG=$(cat ./version/version) build' ] 136 | 137 | - put: docker-image 138 | inputs: [ image ] 139 | params: 140 | image: image/image.tar 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

cr 📂

2 | 3 |
The concurrent runner
4 | 5 |
6 | 7 | [![Build Status](https://hush-house.concourse-ci.org/api/v1/teams/main/pipelines/cr/jobs/test/badge)](https://hush-house.concourse-ci.org/teams/main/pipelines/cr) 8 | 9 | ### Overview 10 | 11 | `cr` is a job executor concerned with achieving the highest parallel execution possible. 12 | 13 | Given a definition of jobs and their dependencies, it builds a graph that outlines the execution plan of these jobs. 14 | 15 | For instance, consider the following plan: 16 | 17 | 18 | ```yaml 19 | Jobs: 20 | - Id: 'SayFoo' 21 | Run: 'echo foo' 22 | 23 | - Id: 'SayBaz' 24 | Run: 'echo baz' 25 | DependsOn: [ 'SayFoo' ] 26 | 27 | - Id: 'SayCaz' 28 | Run: 'echo caz' 29 | DependsOn: [ 'SayFoo' ] 30 | ``` 31 | 32 | This plan states that we have 3 jobs to be executed: `SayFoo`, `SayBaz` and `SayCaz` but the last two jobs must only be executed after the first one and in case it succeeds. 33 | 34 | To visualize the execution plan we can run it with `--graph`, which validates the plan and prints out a [dot](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) digraph. 35 | 36 | 37 | ```sh 38 | # Execute CR telling it where the execution 39 | # plan is (execution.yaml) and that it should 40 | # just print the graph and exit. 41 | cr --file ./execution.yaml --graph 42 | 43 | digraph { 44 | compound = "true" 45 | newrank = "true" 46 | subgraph "root" { 47 | "[root] SayFoo" -> "[root] SayBaz" 48 | "[root] SayFoo" -> "[root] SayCaz" 49 | "[root] _root" -> "[root] SayFoo" 50 | } 51 | } 52 | 53 | # If we pipe this to `dot` and than take the output 54 | # of `dot` we can see the visual representation of the 55 | # digraph. 56 | 57 | cr --file ./examples/hello-world.yaml --graph \ 58 | | dot -Tpng > ./assets/hello-world.graph.png 59 | ``` 60 | 61 | ![](./assets/hello-world.graph.png) 62 | 63 | 64 | ### Spec 65 | 66 | 67 | ```yaml 68 | --- 69 | # Configurations that control the runtime environment. 70 | # These are configurations that can be specified via 71 | # the `cr` CLI (cli takes precedence). 72 | Runtime: 73 | LogDirectory: '/tmp' # base directory to use to store log files 74 | Stdout: false # whether all logs should also go to stdout 75 | Directory: './' # default directory to be used as CWD 76 | 77 | 78 | # Map of environment variables to include in every job 79 | # execution. 80 | # This can be be overriden by job-specific environments 81 | Env: 82 | FOO: 'BAR' 83 | 84 | 85 | # Jobs is a list of `Job` objects. 86 | # Each job can have its properties templated 87 | # using results of other jobs, even if they 88 | # depend on the result of a job execution. 89 | Jobs: 90 | - Id: MyJob # name of the job being executed. 91 | Run: 'echo test' # command to run 92 | Directory: '/tmp' # directory to use as cwd in the execution 93 | CaptureOutput: true # whether the output of the task should be stored in `.Output` variable 94 | Env: # Variables to merge into the environment of the command 95 | FOO: 'BAR' 96 | DependsOn: # List of strings specifying jobs that should be executed before this 97 | - 'AnotherJob' # job and that must exit succesfully. 98 | LogFilepath: '/log' # Path to the file where the logs of this execution should be stored. 99 | # By default they're stored under `/tmp/`. 100 | 101 | ``` 102 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alexflint/go-arg v0.0.0-20170330211029-cef6506c97e5 h1:nPLdS1C6uxjd1fhXGxbfd5yfxQP+7pSgw0VuKlDMk+E= 2 | github.com/alexflint/go-arg v0.0.0-20170330211029-cef6506c97e5/go.mod h1:PHxo6ZWOLVMZZgWSAqBynb/KhIqoGO6WKwOVX7rM9dg= 3 | github.com/alexflint/go-scalar v0.0.0-20170216015739-45e5d6cd8605 h1:D6TUHwBqLVCVb2mHQ4Z+nqR7o6cK9EdaIkgWEWUM1xU= 4 | github.com/alexflint/go-scalar v0.0.0-20170216015739-45e5d6cd8605/go.mod h1:dgifnFPveotJNpwJdl1hDPu5vSuqVVUPIr3isfcvgBA= 5 | github.com/cirocosta/cr v0.0.0-20171218233017-dc8c1f40647b h1:h2PrECb6q0TcEMvX6s5BMT2YzD3HXdpz1qurZ5A/IXs= 6 | github.com/cirocosta/cr v0.0.0-20171218233017-dc8c1f40647b/go.mod h1:EzcRKEZKc5dSJSfpx3YehbPnrB3jEoPQAppVCBjL2pc= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/fatih/color v0.0.0-20170926111411-5df930a27be2 h1:40J76vs1Y7oiHFqTrQHQ6A5u8vbXJdLaMkC9iHU/uMw= 10 | github.com/fatih/color v0.0.0-20170926111411-5df930a27be2/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 11 | github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= 12 | github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 13 | github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8= 14 | github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= 15 | github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= 16 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 17 | github.com/hashicorp/terraform v0.0.0-20171212233002-681b2e75875e h1:uxXFweX2d4Jn7TcGCsN4tLQxS77xjZF4eOku+6AyUdg= 18 | github.com/hashicorp/terraform v0.0.0-20171212233002-681b2e75875e/go.mod h1:uN1KUiT7Wdg61fPwsGXQwK3c8PmpIVZrt5Vcb1VrSoM= 19 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 20 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 21 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 22 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 23 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 24 | github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597 h1:hGizH4aMDFFt1iOA4HNKC13lqIBoCyxIjWcAnWIy7aU= 25 | github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 26 | github.com/mattn/go-isatty v0.0.0-20170307163044-57fdcb988a5c h1:AHfQR/s6GNi92TOh+kfGworqDvTxj2rMsS+Hca87nck= 27 | github.com/mattn/go-isatty v0.0.0-20170307163044-57fdcb988a5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 28 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 29 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 30 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 31 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 32 | github.com/rs/zerolog v1.3.0 h1:41wFrny/ZdgqdUXn9bLpgD134OGZSbLv1bLzFMnfzAA= 33 | github.com/rs/zerolog v1.3.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= 34 | github.com/stretchr/testify v1.1.4 h1:ToftOQTytwshuOSj6bDSolVUa3GINfJP/fg3OkkOzQQ= 35 | github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 36 | golang.org/x/sys v0.0.0-20170213225739-e24f485414ae h1:GTtEQDSA+M757ZEFcmC1Z5tEQXeyj0/vKmKeGqKRbP4= 37 | golang.org/x/sys v0.0.0-20170213225739-e24f485414ae/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 38 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 39 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 40 | gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= 41 | gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 42 | -------------------------------------------------------------------------------- /lib/executor_test.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestResolveJobDirectory(t *testing.T) { 11 | var testCases = []struct { 12 | desc string 13 | job *Job 14 | state *RenderState 15 | expected string 16 | shouldError bool 17 | }{ 18 | { 19 | desc: "nil", 20 | shouldError: true, 21 | }, 22 | { 23 | desc: "empty directory uses default", 24 | job: &Job{ 25 | Directory: "", 26 | }, 27 | state: &RenderState{}, 28 | expected: ".", 29 | }, 30 | { 31 | desc: "templated directory", 32 | job: &Job{ 33 | Directory: "/{{ .Jobs.Job1.Output }}", 34 | }, 35 | state: &RenderState{ 36 | Jobs: map[string]*Job{ 37 | "Job1": {Output: "lol"}, 38 | }, 39 | }, 40 | expected: "/lol", 41 | }, 42 | } 43 | 44 | var ( 45 | err error 46 | actual string 47 | 48 | e = Executor{} 49 | ) 50 | 51 | for _, tc := range testCases { 52 | t.Run(tc.desc, func(t *testing.T) { 53 | actual, err = e.ResolveJobDirectory(tc.job, tc.state) 54 | if tc.shouldError { 55 | require.Error(t, err) 56 | return 57 | } 58 | 59 | require.NoError(t, err) 60 | assert.Equal(t, tc.expected, actual) 61 | }) 62 | } 63 | } 64 | 65 | func TestResolveEnvironment(t *testing.T) { 66 | var testCases = []struct { 67 | desc string 68 | job *Job 69 | globalEnv map[string]string 70 | state *RenderState 71 | expected map[string]string 72 | shouldError bool 73 | }{ 74 | { 75 | desc: "nil", 76 | shouldError: true, 77 | }, 78 | { 79 | desc: "empty environment uses none", 80 | job: &Job{}, 81 | state: &RenderState{}, 82 | expected: map[string]string{}, 83 | }, 84 | { 85 | desc: "custom job environment", 86 | job: &Job{ 87 | Env: map[string]string{"FOO": "BAR"}, 88 | }, 89 | state: &RenderState{}, 90 | expected: map[string]string{"FOO": "BAR"}, 91 | }, 92 | { 93 | desc: "custom job environment with global", 94 | job: &Job{ 95 | Env: map[string]string{"FOO": "BAR"}, 96 | }, 97 | globalEnv: map[string]string{"CAZ": "BAZ"}, 98 | state: &RenderState{}, 99 | expected: map[string]string{"FOO": "BAR", "CAZ": "BAZ"}, 100 | }, 101 | { 102 | desc: "just global", 103 | job: &Job{}, 104 | globalEnv: map[string]string{"CAZ": "BAZ"}, 105 | state: &RenderState{}, 106 | expected: map[string]string{"CAZ": "BAZ"}, 107 | }, 108 | { 109 | desc: "job overriding global", 110 | job: &Job{ 111 | Env: map[string]string{"CAZ": "LOL"}, 112 | }, 113 | globalEnv: map[string]string{"CAZ": "BAZ"}, 114 | state: &RenderState{}, 115 | expected: map[string]string{"CAZ": "LOL"}, 116 | }, 117 | { 118 | desc: "custom job environment with templating", 119 | job: &Job{ 120 | Env: map[string]string{"FOO": "{{ .Jobs.Job1.Output }}"}, 121 | }, 122 | state: &RenderState{ 123 | Jobs: map[string]*Job{ 124 | "Job1": {Output: "lol"}, 125 | }, 126 | }, 127 | expected: map[string]string{"FOO": "lol"}, 128 | }, 129 | { 130 | desc: "global environment with templating", 131 | globalEnv: map[string]string{"CAZ": "{{ .Jobs.Job1.Output }}"}, 132 | job: &Job{}, 133 | state: &RenderState{ 134 | Jobs: map[string]*Job{ 135 | "Job1": {Output: "lol"}, 136 | }, 137 | }, 138 | expected: map[string]string{"CAZ": "lol"}, 139 | }, 140 | } 141 | 142 | var ( 143 | err error 144 | actual map[string]string 145 | 146 | e = Executor{ 147 | config: &Config{}, 148 | } 149 | ) 150 | 151 | for _, tc := range testCases { 152 | t.Run(tc.desc, func(t *testing.T) { 153 | e.config.Env = tc.globalEnv 154 | 155 | actual, err = e.ResolveJobEnv(tc.job, tc.state) 156 | if tc.shouldError { 157 | require.Error(t, err) 158 | return 159 | } 160 | 161 | require.NoError(t, err) 162 | require.Equal(t, len(tc.expected), len(actual)) 163 | 164 | for k, _ := range tc.expected { 165 | assert.Equal(t, tc.expected[k], actual[k]) 166 | } 167 | }) 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /lib/types.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "io" 5 | "os/exec" 6 | "time" 7 | ) 8 | 9 | // Execution represents the instantiation of a command 10 | // whose execution can be limited and tracked. 11 | type Execution struct { 12 | Argv []string 13 | ExitCode int 14 | Env map[string]string 15 | Directory string 16 | Stdout io.Writer 17 | Stderr io.Writer 18 | StartTime time.Time 19 | EndTime time.Time 20 | 21 | cmd *exec.Cmd 22 | } 23 | 24 | // RenderState encapsulates the state that can 25 | // be used when templating a given field. 26 | type RenderState struct { 27 | Jobs map[string]*Job 28 | } 29 | 30 | // Config aggregates all the types of cofiguration 31 | // that can be retrieved from a `.cr.yml` configuration 32 | // file. 33 | type Config struct { 34 | 35 | // Runtime contains the CLI and runtime configuration 36 | // to be applied when running `cr`. 37 | Runtime Runtime `yaml:"Runtime"` 38 | 39 | // Env defines environment variables that should 40 | // be applied to every execution 41 | Env map[string]string `yaml:"Env"` 42 | 43 | // Jobs lists the jobs to be executed. 44 | Jobs []*Job `yaml:"Jobs"` 45 | 46 | // OnJobStatusChange is a callback function to be called 47 | // once per transition of job status. 48 | OnJobStatusChange func(a *Activity) `yaml:"-"` 49 | } 50 | 51 | // Runtime aggragates CLI and runtime configuration 52 | // to be applied when running `cr` 53 | type Runtime struct { 54 | // File denotes the path to the configuration file to 55 | // load 56 | File string `arg:"help:path the configuration file" yaml:"File"` 57 | 58 | // LogsDirectory indicates the path to the directory where logs 59 | // are sent to. 60 | LogsDirectory string `arg:"--logs-directory,help:path to the directory where logs are sent to" yaml:"LogsDirectory"` 61 | 62 | // Stdout indicates whether the execution logs should be pipped 63 | // to stdout or not. 64 | Stdout bool `arg:"help:log executions to stdout" yaml:"Stdout"` 65 | 66 | // Graph indicates whether a dot graph should be output 67 | // or not. 68 | Graph bool `arg:"help:output the execution graph" yaml:"Graph"` 69 | 70 | // Directory denotes what to used as a current working directory 71 | // for the executions when a relative path is indicated in the 72 | // job description. 73 | Directory string `arg:"help:directory to be used as current working directory" yaml:"Directory"` 74 | } 75 | 76 | // Job defines a unit of execution that at some point 77 | // in time gets its command defined in `run` executed. 78 | // It might happen to never be executed if a dependency 79 | // is never met. 80 | type Job struct { 81 | 82 | // Name is the name of the job being executed 83 | Id string `yaml:"Id"` 84 | 85 | // Run is a command to execute in the context 86 | // of a default shell. 87 | Run string `yaml:"Run"` 88 | 89 | // Directory names the absolute or relative path 90 | // to get into before executin the command. 91 | // By default it takes the value "." (current working 92 | // directory). 93 | Directory string `yaml:"Directory"` 94 | 95 | // Whether the output of the execution should 96 | // be stored or not. 97 | CaptureOutput bool `yaml:"CaptureOutput"` 98 | 99 | // Env stores the extra environment to add to the 100 | // command execution. 101 | Env map[string]string `yaml:"Env"` 102 | 103 | // StartTime is the timestamp at the moment of 104 | // the initiation of the execution of the 105 | // command. 106 | StartTime *time.Time `yaml:"-"` 107 | 108 | // EndTime is the timestamp of the end execution 109 | // of the command. 110 | EndTime *time.Time `yaml:"-"` 111 | 112 | // ExitCode stores the result exit-code of the command. 113 | ExitCode int `yaml:"-"` 114 | 115 | // Output is the output captured once the command 116 | // has been executed. 117 | Output string `yaml:"-"` 118 | 119 | // DependsOn lists a series of jobs that the job depends 120 | // on to start its execution. 121 | DependsOn []string `yaml:"DependsOn,flow"` 122 | 123 | // LogFilepath indicates the path to the file where the logs 124 | // of the job execution are sent to. 125 | LogFilepath string `yaml:"LogFilepath"` 126 | } 127 | 128 | func (j Job) Name() string { 129 | return j.Id 130 | } 131 | -------------------------------------------------------------------------------- /lib/names.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "math/rand" 5 | ) 6 | 7 | var ( 8 | left = [...]string{ 9 | "admiring", 10 | "adoring", 11 | "affectionate", 12 | "agitated", 13 | "amazing", 14 | "angry", 15 | "awesome", 16 | "blissful", 17 | "boring", 18 | "brave", 19 | "clever", 20 | "cocky", 21 | "compassionate", 22 | "competent", 23 | "condescending", 24 | "confident", 25 | "cranky", 26 | "dazzling", 27 | "determined", 28 | "distracted", 29 | "dreamy", 30 | "eager", 31 | "ecstatic", 32 | "elastic", 33 | "elated", 34 | "elegant", 35 | "eloquent", 36 | "epic", 37 | "fervent", 38 | "festive", 39 | "flamboyant", 40 | "focused", 41 | "friendly", 42 | "frosty", 43 | "gallant", 44 | "gifted", 45 | "goofy", 46 | "gracious", 47 | "happy", 48 | "hardcore", 49 | "heuristic", 50 | "hopeful", 51 | "hungry", 52 | "infallible", 53 | "inspiring", 54 | "jolly", 55 | "jovial", 56 | "keen", 57 | "kind", 58 | "laughing", 59 | "loving", 60 | "lucid", 61 | "mystifying", 62 | "modest", 63 | "musing", 64 | "naughty", 65 | "nervous", 66 | "nifty", 67 | "nostalgic", 68 | "objective", 69 | "optimistic", 70 | "peaceful", 71 | "pedantic", 72 | "pensive", 73 | "practical", 74 | "priceless", 75 | "quirky", 76 | "quizzical", 77 | "relaxed", 78 | "reverent", 79 | "romantic", 80 | "sad", 81 | "serene", 82 | "sharp", 83 | "silly", 84 | "sleepy", 85 | "stoic", 86 | "stupefied", 87 | "suspicious", 88 | "tender", 89 | "thirsty", 90 | "trusting", 91 | "unruffled", 92 | "upbeat", 93 | "vibrant", 94 | "vigilant", 95 | "vigorous", 96 | "wizardly", 97 | "wonderful", 98 | "xenodochial", 99 | "youthful", 100 | "zealous", 101 | "zen", 102 | } 103 | 104 | right = [...]string{ 105 | "albattani", 106 | 107 | "allen", 108 | 109 | "almeida", 110 | 111 | "agnesi", 112 | 113 | "archimedes", 114 | 115 | "ardinghelli", 116 | 117 | "aryabhata", 118 | 119 | "austin", 120 | 121 | "babbage", 122 | 123 | "banach", 124 | 125 | "bardeen", 126 | 127 | "bartik", 128 | 129 | "bassi", 130 | 131 | "beaver", 132 | 133 | "bell", 134 | 135 | "benz", 136 | 137 | "bhabha", 138 | 139 | "bhaskara", 140 | 141 | "blackwell", 142 | 143 | "bohr", 144 | 145 | "booth", 146 | 147 | "borg", 148 | 149 | "bose", 150 | 151 | "boyd", 152 | 153 | "brahmagupta", 154 | 155 | "brattain", 156 | 157 | "brown", 158 | 159 | "carson", 160 | 161 | "chandrasekhar", 162 | 163 | "chatterjee", 164 | 165 | "shannon", 166 | 167 | "clarke", 168 | 169 | "colden", 170 | 171 | "cori", 172 | 173 | "cray", 174 | 175 | "curran", 176 | 177 | "curie", 178 | 179 | "darwin", 180 | 181 | "davinci", 182 | 183 | "dijkstra", 184 | 185 | "dubinsky", 186 | 187 | "easley", 188 | 189 | "edison", 190 | 191 | "einstein", 192 | 193 | "elion", 194 | 195 | "engelbart", 196 | 197 | "euclid", 198 | 199 | "euler", 200 | 201 | "fermat", 202 | 203 | "fermi", 204 | 205 | "feynman", 206 | 207 | "franklin", 208 | 209 | "galileo", 210 | 211 | "gates", 212 | 213 | "goldberg", 214 | 215 | "goldstine", 216 | 217 | "goldwasser", 218 | 219 | "golick", 220 | 221 | "goodall", 222 | 223 | "haibt", 224 | 225 | "hamilton", 226 | 227 | "hawking", 228 | 229 | "heisenberg", 230 | 231 | "hermann", 232 | 233 | "heyrovsky", 234 | 235 | "hodgkin", 236 | 237 | "hoover", 238 | 239 | "hopper", 240 | 241 | "hugle", 242 | 243 | "hypatia", 244 | 245 | "jackson", 246 | 247 | "jang", 248 | 249 | "jennings", 250 | 251 | "jepsen", 252 | 253 | "johnson", 254 | 255 | "joliot", 256 | 257 | "jones", 258 | 259 | "kalam", 260 | 261 | "kare", 262 | 263 | "keller", 264 | 265 | "kepler", 266 | 267 | "khorana", 268 | 269 | "kilby", 270 | 271 | "kirch", 272 | 273 | "knuth", 274 | 275 | "kowalevski", 276 | 277 | "lalande", 278 | 279 | "lamarr", 280 | 281 | "lamport", 282 | 283 | "leakey", 284 | 285 | "leavitt", 286 | 287 | "lewin", 288 | 289 | "lichterman", 290 | 291 | "liskov", 292 | 293 | "lovelace", 294 | 295 | "lumiere", 296 | 297 | "mahavira", 298 | 299 | "mayer", 300 | 301 | "mccarthy", 302 | 303 | "mcclintock", 304 | 305 | "mclean", 306 | 307 | "mcnulty", 308 | 309 | "meitner", 310 | 311 | "meninsky", 312 | 313 | "mestorf", 314 | 315 | "minsky", 316 | 317 | "mirzakhani", 318 | 319 | "morse", 320 | 321 | "murdock", 322 | 323 | "neumann", 324 | 325 | "newton", 326 | 327 | "nightingale", 328 | 329 | "nobel", 330 | 331 | "noether", 332 | 333 | "northcutt", 334 | 335 | "noyce", 336 | 337 | "panini", 338 | 339 | "pare", 340 | 341 | "pasteur", 342 | 343 | "payne", 344 | 345 | "perlman", 346 | 347 | "pike", 348 | 349 | "poincare", 350 | 351 | "poitras", 352 | 353 | "ptolemy", 354 | 355 | "raman", 356 | 357 | "ramanujan", 358 | 359 | "ride", 360 | 361 | "montalcini", 362 | 363 | "ritchie", 364 | 365 | "roentgen", 366 | 367 | "rosalind", 368 | 369 | "saha", 370 | 371 | "sammet", 372 | 373 | "shaw", 374 | 375 | "shirley", 376 | 377 | "shockley", 378 | 379 | "sinoussi", 380 | 381 | "snyder", 382 | 383 | "spence", 384 | 385 | "stallman", 386 | 387 | "stonebraker", 388 | 389 | "swanson", 390 | 391 | "swartz", 392 | 393 | "swirles", 394 | 395 | "tereshkova", 396 | 397 | "tesla", 398 | 399 | "thompson", 400 | 401 | "torvalds", 402 | 403 | "turing", 404 | 405 | "varahamihira", 406 | 407 | "vaughan", 408 | 409 | "visvesvaraya", 410 | 411 | "volhard", 412 | 413 | "villani", 414 | 415 | "wescoff", 416 | 417 | "wiles", 418 | 419 | "williams", 420 | 421 | "wilson", 422 | 423 | "wing", 424 | 425 | "wozniak", 426 | 427 | "wright", 428 | 429 | "yalow", 430 | 431 | "yonath", 432 | } 433 | ) 434 | 435 | // GetRandomName returns a "dockerish" random name. 436 | // Extracted from https://github.com/moby/moby/tree/5dc791c2debd561f61f04ec5a947f261fe79b275/pkg/namesgenerator 437 | func GetRandomName() string { 438 | return left[rand.Intn(len(left))] + 439 | "_" + 440 | right[rand.Intn(len(right))] 441 | } 442 | -------------------------------------------------------------------------------- /lib/executor.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "os" 8 | "path" 9 | "strings" 10 | "time" 11 | 12 | "github.com/hashicorp/terraform/dag" 13 | "github.com/pkg/errors" 14 | "github.com/rs/zerolog" 15 | ) 16 | 17 | // Executor encapsulates the execution 18 | // context of a graph of jobs. 19 | type Executor struct { 20 | config *Config 21 | graph *dag.AcyclicGraph 22 | logger zerolog.Logger 23 | jobsMap map[string]*Job 24 | logsDirectory string 25 | } 26 | 27 | // New instantiates a new Executor from 28 | // the supplied configuration. 29 | func New(cfg *Config) (e Executor, err error) { 30 | var finfo os.FileInfo 31 | 32 | if cfg == nil { 33 | err = errors.Errorf("cfg must be non-nill") 34 | return 35 | } 36 | 37 | graph, err := BuildDependencyGraph(cfg.Jobs) 38 | if err != nil { 39 | err = errors.Wrapf(err, 40 | "failed to create dependency graph") 41 | return 42 | } 43 | 44 | if cfg.Runtime.LogsDirectory == "" { 45 | err = errors.Errorf("LogsDirectory must be specified") 46 | return 47 | } 48 | 49 | finfo, err = os.Stat(cfg.Runtime.LogsDirectory) 50 | if err != nil { 51 | err = errors.Wrapf(err, 52 | "failed to look for logs directory %s", 53 | cfg.Runtime.LogsDirectory) 54 | return 55 | } 56 | 57 | if !finfo.IsDir() { 58 | err = errors.Errorf( 59 | "logs directory must be a directory %s", 60 | cfg.Runtime.LogsDirectory) 61 | return 62 | } 63 | 64 | e.logsDirectory = cfg.Runtime.LogsDirectory 65 | e.config = cfg 66 | e.graph = &graph 67 | e.jobsMap = map[string]*Job{} 68 | e.logger = zerolog.New(os.Stdout). 69 | With(). 70 | Str("from", "executor"). 71 | Logger() 72 | 73 | for _, job := range cfg.Jobs { 74 | e.jobsMap[job.Id] = job 75 | } 76 | 77 | return 78 | } 79 | 80 | // GetDotGraph retrieves a `dot` visualization of 81 | // the dependency graph. 82 | func (e *Executor) GetDotGraph() (res string) { 83 | res = string(e.graph.Dot(&dag.DotOpts{})) 84 | return 85 | } 86 | 87 | // Execute initiates the parallel execution of the 88 | // jobs. 89 | func (e *Executor) Execute(ctx context.Context) (err error) { 90 | err = e.TraverseAndExecute(ctx, e.graph) 91 | if err != nil { 92 | err = errors.Wrapf(err, "jobs execution failed") 93 | return 94 | } 95 | 96 | return 97 | } 98 | 99 | func (e *Executor) ResolveJobDirectory(j *Job, renderState *RenderState) (res string, err error) { 100 | if j == nil || renderState == nil { 101 | err = errors.Errorf("job and renderState must be non-nil") 102 | return 103 | } 104 | 105 | switch j.Directory { 106 | case "": 107 | res = "." 108 | default: 109 | res, err = TemplateField(j.Directory, renderState) 110 | if err != nil { 111 | err = errors.Wrapf(err, 112 | "couldn't render Directory string") 113 | return 114 | } 115 | } 116 | 117 | return 118 | } 119 | 120 | func (e *Executor) ResolveJobLogFilepath(j *Job, renderState *RenderState) (res string, err error) { 121 | if j == nil || renderState == nil { 122 | err = errors.Errorf("job and renderState must be non-nil") 123 | return 124 | } 125 | 126 | switch j.LogFilepath { 127 | case "": 128 | res = path.Join( 129 | e.config.Runtime.LogsDirectory, 130 | j.Id) 131 | default: 132 | res, err = TemplateField(j.LogFilepath, renderState) 133 | if err != nil { 134 | err = errors.Wrapf(err, 135 | "couldn't render LogFilepath string") 136 | return 137 | } 138 | } 139 | 140 | return 141 | } 142 | 143 | func (e *Executor) ResolveJobRun(j *Job, renderState *RenderState) (res string, err error) { 144 | if j == nil || renderState == nil { 145 | err = errors.Errorf("job and renderState must be non-nil") 146 | return 147 | } 148 | 149 | switch j.Run { 150 | case "": 151 | res = "" 152 | default: 153 | res, err = TemplateField(j.Run, renderState) 154 | if err != nil { 155 | err = errors.Wrapf(err, 156 | "couldn't render run command") 157 | return 158 | } 159 | } 160 | 161 | return 162 | } 163 | 164 | func (e *Executor) ResolveJobEnv(j *Job, renderState *RenderState) (res map[string]string, err error) { 165 | res = map[string]string{} 166 | 167 | if j == nil || renderState == nil { 168 | err = errors.Errorf("job and renderState must be non-nil") 169 | return 170 | } 171 | 172 | var templateRes string 173 | for k, v := range e.config.Env { 174 | templateRes, err = TemplateField(v, renderState) 175 | if err != nil { 176 | err = errors.Errorf( 177 | "failed to template environment variable %s", k) 178 | return 179 | } 180 | 181 | res[k] = templateRes 182 | } 183 | 184 | for k, v := range j.Env { 185 | templateRes, err = TemplateField(v, renderState) 186 | if err != nil { 187 | err = errors.Errorf( 188 | "failed to template environment variable %s", k) 189 | return 190 | } 191 | 192 | res[k] = templateRes 193 | } 194 | 195 | return 196 | } 197 | 198 | // RunJob is a method invoked for each vertex 199 | // in the execution graph except the root. 200 | // TODO Split into a job preparation step and a 201 | // job execution step. 202 | func (e *Executor) RunJob(ctx context.Context, j *Job) (err error) { 203 | var ( 204 | execution *Execution 205 | logFile *os.File 206 | output bytes.Buffer 207 | 208 | stdout = []io.Writer{} 209 | stderr = []io.Writer{} 210 | renderState = &RenderState{ 211 | Jobs: e.jobsMap, 212 | } 213 | ) 214 | 215 | if j.CaptureOutput { 216 | stdout = append(stdout, &output) 217 | } 218 | 219 | if e.config.Runtime.Stdout { 220 | stdout = append(stdout, os.Stdout) 221 | stderr = append(stdout, os.Stderr) 222 | } 223 | 224 | j.LogFilepath, err = e.ResolveJobLogFilepath(j, renderState) 225 | if err != nil { 226 | return 227 | } 228 | 229 | logFile, err = os.Create(j.LogFilepath) 230 | if err != nil { 231 | err = errors.Wrapf(err, 232 | "failed to create file for logging %s", 233 | j.LogFilepath) 234 | return 235 | } 236 | defer logFile.Close() 237 | 238 | stdout = append(stdout, logFile) 239 | stderr = append(stderr, logFile) 240 | 241 | j.Directory, err = e.ResolveJobDirectory(j, renderState) 242 | if err != nil { 243 | return 244 | } 245 | 246 | j.Env, err = e.ResolveJobEnv(j, renderState) 247 | if err != nil { 248 | return 249 | } 250 | 251 | j.Run, err = e.ResolveJobRun(j, renderState) 252 | if err != nil { 253 | return 254 | } 255 | 256 | if j.Run == "" { 257 | goto END 258 | } 259 | 260 | execution = &Execution{ 261 | Argv: []string{ 262 | "/bin/bash", 263 | "-c", 264 | j.Run, 265 | }, 266 | Stdout: io.MultiWriter(stdout...), 267 | Stderr: io.MultiWriter(stderr...), 268 | Directory: j.Directory, 269 | Env: j.Env, 270 | } 271 | 272 | if e.config.OnJobStatusChange != nil { 273 | e.config.OnJobStatusChange(&Activity{ 274 | Type: ActivityStarted, 275 | Time: time.Now(), 276 | Job: j, 277 | }) 278 | } 279 | 280 | err = execution.Run(ctx) 281 | 282 | j.StartTime = &execution.StartTime 283 | j.EndTime = &execution.EndTime 284 | 285 | if err != nil { 286 | err = errors.Wrapf(err, "command execution failed") 287 | 288 | if e.config.OnJobStatusChange != nil { 289 | e.config.OnJobStatusChange(&Activity{ 290 | Type: ActivityErrored, 291 | Time: time.Now(), 292 | Job: j, 293 | }) 294 | } 295 | 296 | return 297 | } 298 | 299 | j.Output = strings.TrimSpace(output.String()) 300 | 301 | END: 302 | if e.config.OnJobStatusChange != nil { 303 | e.config.OnJobStatusChange(&Activity{ 304 | Type: ActivitySuccess, 305 | Time: time.Now(), 306 | Job: j, 307 | }) 308 | } 309 | 310 | return 311 | } 312 | 313 | func (e *Executor) CreateWalkFunc(ctx context.Context) dag.WalkFunc { 314 | return func(v dag.Vertex) error { 315 | job, ok := v.(*Job) 316 | if !ok { 317 | return errors.Errorf("vertex not a job") 318 | } 319 | 320 | if job.Id == "_root" { 321 | return nil 322 | } 323 | 324 | return e.RunJob(ctx, job) 325 | } 326 | } 327 | 328 | // TraverseAndExecute goes through the graph 329 | // provided and starts the execution of the jobs. 330 | func (e *Executor) TraverseAndExecute(ctx context.Context, g *dag.AcyclicGraph) (err error) { 331 | w := &dag.Walker{ 332 | Callback: e.CreateWalkFunc(ctx), 333 | } 334 | 335 | w.Update(g) 336 | 337 | err = w.Wait() 338 | if err != nil { 339 | err = errors.Wrapf(err, 340 | "execution of jobs failed") 341 | return 342 | } 343 | 344 | return 345 | } 346 | --------------------------------------------------------------------------------