├── .travis.yml ├── CHANGELOG.md ├── DEVELOPING.md ├── LANGUAGE.md ├── LICENSE ├── README.md ├── assertion.go ├── assertion_test.go ├── assoc ├── assoc.go ├── dydbassoc │ ├── dydbassoc.go │ ├── dydbassoc_test.go │ └── scanner.go ├── kind_string.go └── test │ ├── fakeassoc.go │ └── inmemassoc.go ├── batch ├── batch.go ├── flags.go ├── flags_test.go └── state_string.go ├── blob ├── blob.go ├── mux.go ├── nopblob │ └── nopblob.go ├── s3blob │ ├── s3blob.go │ └── s3blob_test.go └── testblob │ ├── mux_test.go │ └── testblob.go ├── bootstrap ├── bootstrap.go ├── client │ └── client.go └── common │ └── image.go ├── cache.go ├── cmd ├── buildreflow │ └── build.go ├── ec2instances │ └── main.go ├── genmetrics │ └── main.go ├── reflow │ ├── dynamodb.go │ ├── ec2.go │ ├── main.go │ └── s3.go └── reflowbootstrap │ ├── install.bash │ └── main.go ├── config.md ├── constants.go ├── context └── context.go ├── digester.go ├── doc.go ├── doc ├── 1000align │ ├── 1000align.rf │ ├── align.rf │ ├── bam.rf │ └── fastq.rf ├── guide.md ├── images │ ├── annotated_trace_overview.jpeg │ ├── selection_tool.jpeg │ └── timing_tool.jpeg └── tracing.md ├── ec2authenticator ├── ec2authenticator.go └── ec2authenticator_test.go ├── ec2cluster ├── amis.go ├── cloudconfig.go ├── cloudconfig_test.go ├── ec2_test.go ├── ec2cluster.go ├── ec2cluster_test.go ├── ec2instance.go ├── ec2spot.go ├── ecr_test.go ├── instance.go ├── instance_test.go ├── instances │ ├── instances.go │ ├── util.go │ ├── util_test.go │ └── verified.go ├── instancestate.go ├── instancestate_test.go ├── manager.go ├── manager_test.go ├── setup.go ├── spotaz.go ├── spotaz_test.go ├── spotprober.go ├── spotprober_test.go ├── stats.go ├── test │ └── fakecluster.go ├── tmpl_test.go └── volume │ ├── device.go │ ├── ec2volume_test.go │ ├── volume.go │ ├── volume_test.go │ ├── watcher.go │ └── watcher_test.go ├── errors ├── errors.go └── errors_test.go ├── executor.go ├── executor_test.go ├── export_test.go ├── fileset.go ├── fileset.pb.go ├── fileset.proto ├── fileset_test.go ├── filesetjson.go ├── flow ├── assoc_test.go ├── context.go ├── context_test.go ├── dot.go ├── eval.go ├── eval_test.go ├── export_test.go ├── fileset.go ├── flow.go ├── flow_test.go ├── mutation_string.go ├── op_string.go ├── prefixwriter.go ├── prefixwriter_test.go ├── repair.go ├── state_string.go ├── stats.go ├── stats_test.go ├── strutil.go ├── strutil_test.go └── workingset.go ├── go.mod ├── go.sum ├── infra ├── infra.go └── infra_test.go ├── internal ├── ecrauth │ └── ecrauth.go ├── execimage │ └── execimage.go ├── fs │ └── fs_unix.go ├── iputil │ └── external.go ├── s3walker │ ├── s3walker.go │ └── s3walker_test.go ├── scanner │ ├── README │ ├── example_test.go │ ├── scanner.go │ └── scanner_test.go └── walker │ ├── walker.go │ └── walker_test.go ├── lang ├── doc.go ├── error.go ├── lex.go ├── lex_test.go ├── op_string.go ├── prog.go ├── prog_test.go ├── reflow.go ├── reflow.y ├── reflow_test.go ├── template.go ├── template_test.go ├── testdata │ ├── escape.reflow │ ├── hash.reflow │ ├── intern.reflow │ ├── internlist.reflow │ ├── internlist2.reflow │ ├── internlistrewrite.reflow │ ├── internrewrite.reflow │ ├── main.reflow │ ├── param.reflow │ └── sub.reflow ├── type.go ├── type_test.go └── y.go ├── liveset ├── bloomlive │ └── bloomlive.go └── liveset.go ├── local ├── blob.go ├── blob_test.go ├── cloudwatch.go ├── cloudwatch_test.go ├── cpufeatures_fallback.go ├── cpufeatures_linux.go ├── docker.go ├── dockerutil.go ├── dockerutil_test.go ├── exec.go ├── execstate_string.go ├── executor.go ├── executor_test.go ├── localfile.go ├── manifest.go ├── oom.go ├── oom_test.go ├── pool.go ├── pool_test.go ├── pooltest │ ├── doc.go │ └── pool_test.go ├── stats.go ├── stats_test.go ├── testutil │ └── pool.go └── zombie.go ├── localcluster ├── localcluster.go └── localcluster_test.go ├── log ├── log.go └── log_test.go ├── metrics ├── README.md ├── client.go ├── metrics.go ├── metrics.yaml ├── monitoring │ ├── metricnames.go │ ├── monitor.go │ ├── monitor_test.go │ ├── oomdetector.go │ └── oomdetector_test.go ├── nopclient.go └── prometrics │ └── client.go ├── pool ├── alloc.go ├── cachingpool.go ├── cachingpool_test.go ├── client │ └── client.go ├── mux.go ├── mux_test.go ├── pool.go ├── pool_test.go ├── resourcepool.go ├── resourcepool_test.go └── server │ ├── server.go │ └── server_test.go ├── predictor ├── predictor.go ├── predictor_test.go ├── taskgroup.go └── taskgroup_test.go ├── reflow.svg ├── reflowlet └── reflowlet.go ├── repository.go ├── repository ├── blobrepo │ ├── blobrepo.go │ ├── blobrepo_test.go │ └── dial.go ├── client │ └── client.go ├── filerepo │ ├── repository.go │ ├── repository_test.go │ └── walker.go ├── http │ └── dial.go ├── manager.go ├── manager_test.go ├── missing.go ├── noprepo │ └── noprepo.go ├── repository.go ├── repository_test.go ├── s3 │ ├── repo.go │ ├── repo_test.go │ └── test │ │ └── fakes3.go ├── server │ ├── server.go │ └── server_test.go └── test │ ├── calltype_string.go │ ├── inmemrepo.go │ ├── manager_test.go │ └── repository.go ├── resources.go ├── resources_test.go ├── rest ├── client.go ├── rest_test.go ├── server.go └── server_test.go ├── runner ├── cluster.go ├── phase_string.go └── runner.go ├── runtime ├── eval.go ├── resolver.go ├── resolver_test.go ├── runflags.go ├── runflags_test.go ├── runner.go ├── runner_test.go ├── runtime.go ├── scheduler.go ├── scheduler_test.go └── util.go ├── sched ├── alloc.go ├── export_test.go ├── internal │ ├── execstate.go │ ├── execstate_test.go │ └── utiltest │ │ └── utiltest.go ├── scale │ └── scheduler_scale_test.go ├── scheduler.go ├── scheduler_test.go ├── stats.go └── task.go ├── syntax ├── abbrev_test.go ├── bundle.go ├── bundle_test.go ├── coerce.go ├── compat.go ├── decl.go ├── digest.go ├── digest_test.go ├── doc.go ├── error.go ├── eval.go ├── eval_test.go ├── eval_test │ ├── eval_test.go │ └── testdata │ │ ├── arith.rf │ │ ├── builtin_override.rf │ │ ├── compare.rf │ │ ├── compr.rf │ │ ├── delayed.rf │ │ ├── dirs.rf │ │ ├── dirs_err.rf │ │ ├── dirs_err2.rf │ │ ├── files.rf │ │ ├── flag_dependence1.rf │ │ ├── flag_dependence2.rf │ │ ├── flag_dependence3.rf │ │ ├── float.rf │ │ ├── fold.rf │ │ ├── if.rf │ │ ├── list_compr_err.rf │ │ ├── map_compr_err.rf │ │ ├── missingnewline.rf │ │ ├── mul.rf │ │ ├── newmodule.rf │ │ ├── oldmodule.reflow │ │ ├── path.rf │ │ ├── prec.rf │ │ ├── reduce.rf │ │ ├── regexp.rf │ │ ├── strings.rf │ │ ├── strings_err1.rf │ │ ├── strings_err2.rf │ │ ├── strings_err3.rf │ │ ├── switch.rf │ │ ├── test1.rf │ │ ├── test_flag_dependence.rf │ │ ├── testdir │ │ ├── 0 │ │ ├── 1 │ │ ├── 2 │ │ ├── 3 │ │ ├── 4 │ │ ├── 5 │ │ ├── 6 │ │ ├── a │ │ ├── aa │ │ ├── aaa │ │ ├── aaab │ │ ├── ab │ │ ├── ac │ │ ├── ad │ │ ├── b │ │ ├── c │ │ ├── d │ │ ├── e │ │ ├── f │ │ └── g │ │ ├── testdir2 │ │ ├── hello.txt │ │ └── world.txt │ │ ├── typealias.rf │ │ └── typealias2.rf ├── expand.go ├── expr.go ├── force.go ├── force_test.go ├── module.go ├── module_test.go ├── parse.go ├── parse_test.go ├── pat.go ├── pat_test.go ├── reflow.y ├── reflow.y.go ├── reflow.y.output ├── registration.go ├── repeat.go ├── repeat_test.go ├── reqs_test.go ├── session.go ├── stdlib.go ├── switch.go ├── switch_test.go ├── testdata │ ├── bundle │ │ ├── concat.rf │ │ ├── main.rf │ │ ├── param.rf │ │ └── parammain.rf │ ├── constmodule.rf │ ├── err1.rf │ ├── err2.rf │ ├── err3.rf │ ├── err4.rf │ ├── err5.rf │ ├── err6.rf │ ├── err7.rf │ ├── err8.rf │ ├── flag.rf │ ├── flag1.rf │ ├── flag_dependence1.rf │ ├── flag_dependence2.rf │ ├── flag_dependence3.rf │ ├── flow.rf │ ├── imagewarn.rf │ ├── instantiate.rf │ ├── lib.rf │ ├── req.rf │ ├── typerr1.rf │ ├── typerr10a.rf │ ├── typerr10b.rf │ ├── typerr10c.rf │ ├── typerr11.rf │ ├── typerr12.rf │ ├── typerr13.rf │ ├── typerr14a.rf │ ├── typerr14b.rf │ ├── typerr14c.rf │ ├── typerr15.rf │ ├── typerr16.rf │ ├── typerr17.rf │ ├── typerr18.rf │ ├── typerr19.rf │ ├── typerr2.rf │ ├── typerr20.rf │ ├── typerr21.rf │ ├── typerr3.rf │ ├── typerr4.rf │ ├── typerr4mod.rf │ ├── typerr5.reflow │ ├── typerr5.rf │ ├── typerr6.rf │ ├── typerr7.rf │ ├── typerr8.rf │ ├── typerr9.rf │ └── unusedwarn.rf ├── type_test.go └── v0module.go ├── taskdb ├── dynamodbtask │ ├── dynamodb.go │ ├── dynamodb_test.go │ ├── scanner.go │ └── setup.go ├── inmemorytaskdb │ └── inmemorytaskdb.go ├── noptaskdb │ └── noptaskdb.go ├── taskdb.go └── taskdb_test.go ├── test ├── flow │ └── constructor.go ├── infra │ └── test.go ├── regress │ ├── fake.go │ ├── regress_test.go │ └── testdata │ │ ├── bridge.rf │ │ ├── coerce.rf │ │ ├── flagparam.rf │ │ ├── generate.rf │ │ ├── inputdata │ │ ├── mapdigest.rf │ │ ├── oldmodule.reflow │ │ ├── repeatexec.rf │ │ └── repeatexecskip.rf └── testutil │ ├── assoc.go │ ├── cache.go │ ├── eval.go │ ├── executor.go │ ├── flow.go │ ├── fuzz.go │ ├── reflowtestrunner.go │ ├── reflowtestrunner_test.go │ ├── repository.go │ ├── repositorycallkind_string.go │ ├── skipifnocreds.go │ ├── taskdb.go │ ├── testutil.go │ └── transferer.go ├── tool ├── batch.go ├── batch_test.go ├── bundle.go ├── cache.go ├── cat.go ├── check.go ├── cmd.go ├── collect.go ├── collect_test.go ├── config.go ├── cost.go ├── cost_test.go ├── doc.go ├── ec2.go ├── ec2verify.go ├── ec2verify_test.go ├── http.go ├── httputil.go ├── images.go ├── info.go ├── info_test.go ├── kill.go ├── limit.go ├── list.go ├── logs.go ├── main.go ├── pred.go ├── ps.go ├── repair.go ├── run.go ├── serve.go ├── shell.go ├── sync.go ├── testdata │ ├── config.json │ ├── samples.csv │ └── test.rf ├── upgrade.go └── version.go ├── trace ├── kind_string.go ├── localtrace │ ├── localtracer.go │ ├── localtracer_scale_test.go │ └── localtracer_test.go ├── trace.go ├── trace_test.go ├── tracer.go └── xraytrace │ ├── xray.go │ └── xray_test.go ├── types ├── constlevel_string.go ├── symtab.go ├── type_test.go └── types.go ├── util.go ├── values ├── digest_test.go ├── dir_test.go ├── env.go ├── env_test.go ├── equal_test.go ├── less_test.go ├── map_test.go ├── pretty_test.go └── values.go └── wg ├── wg.go └── wg_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | jobs: 3 | include: 4 | - os: linux 5 | dist: trusty 6 | go: 1.16.x 7 | - os: osx 8 | osx_image: xcode11.2 9 | go: 1.16.x 10 | # TODO(prasadgopal): running of master is broken as of 12/17/2019. 11 | # Re evaluate if we should even test of golang master. 12 | # - os: linux 13 | # dist: trusty 14 | # go: master 15 | # - os: osx 16 | # osx_image: xcode11.2 17 | # go: master 18 | install: true 19 | script: 20 | - env GO111MODULE=on go test -v -tags unit ./... 21 | -------------------------------------------------------------------------------- /DEVELOPING.md: -------------------------------------------------------------------------------- 1 | # Development notes 2 | 3 | This document contains various notes for working with Reflow's code base. 4 | 5 | # Running a local Reflow server 6 | 7 | It's often useful to run a local standing Reflow server, instead of using -local, 8 | e.g., when debugging interactions between the Reflow evaluator and 9 | a Reflow server, or debugging work stealing. 10 | 11 | First, launch a local Reflow server. (If you launch multiple instances, make sure 12 | they have different working directories.) 13 | 14 | % reflow serve -addr :9000 -config /tmp/config -dir /tmp/reflowlet 15 | 16 | And then use the "static" cluster provider to use them from a Reflow 17 | invocation, e.g.,: 18 | 19 | % reflow -config /tmp/config -cluster static,localhost:9000 ... 20 | 21 | Multiple instances are separated by commas. 22 | -------------------------------------------------------------------------------- /assoc/kind_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Kind"; DO NOT EDIT. 2 | 3 | package assoc 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[Fileset-0] 12 | _ = x[FilesetV2-1] 13 | } 14 | 15 | const _Kind_name = "FilesetFilesetV2" 16 | 17 | var _Kind_index = [...]uint8{0, 7, 16} 18 | 19 | func (i Kind) String() string { 20 | if i < 0 || i >= Kind(len(_Kind_index)-1) { 21 | return "Kind(" + strconv.FormatInt(int64(i), 10) + ")" 22 | } 23 | return _Kind_name[_Kind_index[i]:_Kind_index[i+1]] 24 | } 25 | -------------------------------------------------------------------------------- /assoc/test/fakeassoc.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "github.com/grailbio/infra" 5 | "github.com/grailbio/reflow/assoc" 6 | infra2 "github.com/grailbio/reflow/infra" 7 | ) 8 | 9 | func init() { 10 | infra.Register("fakeassoc", new(Assoc)) 11 | } 12 | 13 | // Assoc is a fake assoc infra provider and should be used only in tests. 14 | type Assoc struct { 15 | assoc.Assoc 16 | infra2.TableNameFlagsTrait 17 | } 18 | -------------------------------------------------------------------------------- /assoc/test/inmemassoc.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/grailbio/infra" 7 | infra2 "github.com/grailbio/reflow/infra" 8 | "github.com/grailbio/reflow/test/testutil" 9 | ) 10 | 11 | func init() { 12 | infra.Register("inmemassoc", new(InMemoryAssoc)) 13 | } 14 | 15 | // InMemoryAssoc is an in-memory assoc infra provider 16 | type InMemoryAssoc struct { 17 | *testutil.InmemoryAssoc 18 | infra2.TableNameFlagsTrait 19 | } 20 | 21 | var ( 22 | mu sync.Mutex 23 | assocs = make(map[string]*testutil.InmemoryAssoc) 24 | ) 25 | 26 | // Init implements infra.Provider 27 | func (a *InMemoryAssoc) Init() error { 28 | mu.Lock() 29 | defer mu.Unlock() 30 | if ea, ok := assocs[a.TableName]; ok { 31 | a.InmemoryAssoc = ea 32 | return nil 33 | } 34 | a.InmemoryAssoc = testutil.NewInmemoryAssoc() 35 | assocs[a.TableName] = a.InmemoryAssoc 36 | return nil 37 | } 38 | 39 | func GetInMemoryAssoc(tableName string) *testutil.InmemoryAssoc { 40 | mu.Lock() 41 | defer mu.Unlock() 42 | return assocs[tableName] 43 | } 44 | -------------------------------------------------------------------------------- /batch/flags.go: -------------------------------------------------------------------------------- 1 | package batch 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | 7 | "github.com/grailbio/reflow/errors" 8 | ) 9 | 10 | func parseFlags(flags *flag.FlagSet, runArgs map[string]string, cmdArgs []string) error { 11 | // Check that all specified runArgs are valid flags. 12 | for k := range runArgs { 13 | if flags.Lookup(k) == nil { 14 | return errors.New("non-existent flag " + k) 15 | } 16 | } 17 | 18 | var err error 19 | flags.VisitAll(func(f *flag.Flag) { 20 | if v, ok := runArgs[f.Name]; ok { 21 | if e := f.Value.Set(v); e != nil { 22 | err = e 23 | } 24 | } 25 | }) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | flags.SetOutput(ioutil.Discard) 31 | flags.Usage = func() {} 32 | return flags.Parse(cmdArgs) 33 | } 34 | -------------------------------------------------------------------------------- /batch/state_string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Code generated by "stringer -type=State"; DO NOT EDIT 6 | 7 | package batch 8 | 9 | import "fmt" 10 | 11 | const _State_name = "StateInitStateRunningStateDoneStateErrorstateMax" 12 | 13 | var _State_index = [...]uint8{0, 9, 21, 30, 40, 48} 14 | 15 | func (i State) String() string { 16 | if i < 0 || i >= State(len(_State_index)-1) { 17 | return fmt.Sprintf("State(%d)", i) 18 | } 19 | return _State_name[_State_index[i]:_State_index[i+1]] 20 | } 21 | -------------------------------------------------------------------------------- /blob/nopblob/nopblob.go: -------------------------------------------------------------------------------- 1 | package nopblob 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "strings" 7 | 8 | "github.com/grailbio/reflow" 9 | "github.com/grailbio/reflow/blob" 10 | ) 11 | 12 | // Store implements blob.Store. 13 | type Store struct { 14 | buckets map[string]*Bucket 15 | } 16 | 17 | func NewStore() *Store { 18 | return &Store{ 19 | buckets: make(map[string]*Bucket), 20 | } 21 | } 22 | 23 | func (s *Store) Bucket(ctx context.Context, bucket string) (blob.Bucket, error) { 24 | return &Bucket{bucket: bucket}, nil 25 | } 26 | 27 | // Bucket implements blob.Bucket. 28 | type Bucket struct { 29 | bucket string 30 | } 31 | 32 | func (b *Bucket) File(ctx context.Context, key string) (reflow.File, error) { 33 | return reflow.File{}, nil 34 | } 35 | 36 | func (b *Bucket) Scan(prefix string, withMetadata bool) blob.Scanner { 37 | return &nopScanner{} 38 | } 39 | 40 | func (b *Bucket) Download(ctx context.Context, key, etag string, size int64, w io.WriterAt) (int64, error) { 41 | return 0, nil 42 | } 43 | 44 | func (b *Bucket) Get(ctx context.Context, key, etag string) (io.ReadCloser, reflow.File, error) { 45 | return io.NopCloser(strings.NewReader("")), reflow.File{}, nil 46 | } 47 | 48 | func (b *Bucket) Put(ctx context.Context, key string, size int64, body io.Reader, contentHash string) error { 49 | return nil 50 | } 51 | 52 | func (b *Bucket) Snapshot(ctx context.Context, prefix string) (reflow.Fileset, error) { 53 | return reflow.Fileset{}, nil 54 | } 55 | 56 | func (b *Bucket) Copy(ctx context.Context, src, dst, contentHash string) error { 57 | return nil 58 | } 59 | 60 | func (b *Bucket) CopyFrom(ctx context.Context, srcBucket blob.Bucket, src, dst string) error { 61 | return nil 62 | } 63 | 64 | func (b *Bucket) Delete(ctx context.Context, keys ...string) error { 65 | return nil 66 | } 67 | 68 | func (b *Bucket) Location() string { 69 | return "" 70 | } 71 | 72 | // nopScanner implements blob.Scanner. 73 | type nopScanner struct{} 74 | 75 | func (s *nopScanner) Scan(ctx context.Context) bool { 76 | return false 77 | } 78 | 79 | func (s *nopScanner) Err() error { 80 | return nil 81 | } 82 | 83 | func (s *nopScanner) File() reflow.File { 84 | return reflow.File{} 85 | } 86 | 87 | func (s *nopScanner) Key() string { 88 | return "" 89 | } 90 | -------------------------------------------------------------------------------- /blob/testblob/mux_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package testblob 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | 11 | "github.com/grailbio/reflow/blob" 12 | ) 13 | 14 | func TestBucketKey(t *testing.T) { 15 | m := blob.Mux{"test": New("test")} 16 | for _, tc := range []struct { 17 | rawurl string 18 | wantkey string 19 | wanterr bool 20 | }{ 21 | {"test://bucket/Helloworld", "Helloworld", false}, 22 | {"test://bucket/Hello #world #now", "Hello #world #now", false}, 23 | {"test://bucket/tmp/reflow/modules/internal/grailrnar/summary_want.tsv?=2", "tmp/reflow/modules/internal/grailrnar/summary_want.tsv?=2", false}, 24 | } { 25 | _, got, err := m.Bucket(context.Background(), tc.rawurl) 26 | if (err != nil) != tc.wanterr { 27 | t.Errorf("got %v want error %v", err, tc.wanterr) 28 | } 29 | if tc.wanterr { 30 | continue 31 | } 32 | if got != tc.wantkey { 33 | t.Errorf("got %v want %v", got, tc.wantkey) 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /bootstrap/common/image.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package common 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "os" 12 | "strings" 13 | "syscall" 14 | ) 15 | 16 | const ExecImageErrPrefix = "bootstrap execimage" 17 | 18 | // Image is an image binary to be installed and executed on the bootstrap. 19 | type Image struct { 20 | Path string `json:"path"` 21 | Name string `json:"name"` 22 | Args []string `json:"args"` 23 | } 24 | 25 | // InstallImage reads a new image from its argument and replaces the current 26 | // process with it. As a consequence, all state held by the caller is lost 27 | // (pending requests, if any, etc) so its up to the caller to manage this interaction. 28 | func InstallImage(image io.ReadCloser, args []string, imageName string) error { 29 | f, err := ioutil.TempFile("", imageName) 30 | if err != nil { 31 | return err 32 | } 33 | if _, err := io.Copy(f, image); err != nil { 34 | return err 35 | } 36 | if err := image.Close(); err != nil { 37 | return err 38 | } 39 | path := f.Name() 40 | if err := f.Close(); err != nil { 41 | return err 42 | } 43 | if err := os.Chmod(path, 0755); err != nil { 44 | return err 45 | } 46 | cmd := append([]string{path}, args...) 47 | if err = syscall.Exec(path, cmd, os.Environ()); err != nil { 48 | return fmt.Errorf("InstallImage %s: %v", strings.Join(cmd, " "), err) 49 | } 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package reflow 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/grailbio/base/digest" 11 | ) 12 | 13 | // A Cache stores Values and their associated File objects for later 14 | // retrieval. Caches may be temporary: objects are not guaranteed 15 | // to persist. 16 | type Cache interface { 17 | // Lookup returns the value associated with a (digest) key. 18 | // Lookup returns an error flagged errors.NotExist when there 19 | // is no such value. 20 | // 21 | // Lookup should also check to make sure that the objects 22 | // actually exist, and provide a reasonable guarantee that they'll 23 | // be available for transfer. 24 | // 25 | // TODO(marius): allow the caller to maintain a lease on the desired 26 | // objects so that garbage collection can (safely) be run 27 | // concurrently with flows. This isn't a correctness concern (the 28 | // flows may be restarted), but rather one of efficiency. 29 | Lookup(context.Context, digest.Digest) (Fileset, error) 30 | 31 | // Transfer transmits the file objects associated with value v 32 | // (usually retrieved by Lookup) to the repository dst. Transfer 33 | // should be used in place of direct (cache) repository access since 34 | // it may apply additional policies (e.g., rate limiting, etc.) 35 | Transfer(ctx context.Context, dst Repository, v Fileset) error 36 | 37 | // NeedTransfer returns the set of files in the Fileset v that are absent 38 | // in the provided repository. 39 | NeedTransfer(ctx context.Context, dst Repository, v Fileset) ([]File, error) 40 | 41 | // Write stores the Value v, whose file objects exist in Repository repo, 42 | // under the key id. If the repository is nil no objects are transferred. 43 | Write(ctx context.Context, id digest.Digest, v Fileset, repo Repository) error 44 | 45 | // Delete removes the value named by id from this cache. 46 | Delete(ctx context.Context, id digest.Digest) error 47 | 48 | // Repository returns this cache's underlying repository. It should 49 | // not be used for data transfer during the course of evaluation; see 50 | // Transfer. 51 | Repository() Repository 52 | } 53 | -------------------------------------------------------------------------------- /cmd/reflow/s3.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | "io/ioutil" 12 | "os" 13 | "strings" 14 | 15 | "github.com/grailbio/reflow/tool" 16 | ) 17 | 18 | func setupS3Repository(c *tool.Cmd, _ context.Context, args ...string) { 19 | flags := flag.NewFlagSet("setup-s3-repository", flag.ExitOnError) 20 | help := `Setup-s3-repository provisions a bucket in AWS's S3 storage service 21 | and modifies Reflow's configuration to use this S3 bucket as its 22 | object repository. 23 | 24 | The resulting configuration can be examined with "reflow config"` 25 | c.Parse(flags, args, help, "setup-s3-repository ") 26 | if flags.NArg() != 1 { 27 | flags.Usage() 28 | } 29 | bucket := flags.Arg(0) 30 | 31 | b, err := ioutil.ReadFile(c.ConfigFile) 32 | if err != nil && !os.IsNotExist(err) { 33 | c.Fatal(err) 34 | } 35 | config, err := c.Schema.Unmarshal(b) 36 | if err != nil { 37 | c.Fatal(err) 38 | } 39 | pkgPath := "s3" 40 | keys := config.Keys 41 | if v, ok := keys["repository"]; ok { 42 | parts := strings.Split(v.(string), ",") 43 | if len(parts) == 1 && parts[0] == "noprepo" { 44 | c.Log.Debugf("replacing noprepo repository with %s", pkgPath) 45 | } else if len(parts) != 2 || parts[0] != pkgPath || parts[1] != bucket { 46 | c.Fatalf("%s repository already set up; cannot set up %s repository", v, pkgPath) 47 | } else { 48 | c.Log.Printf("repository already set up; updating schemas") 49 | } 50 | } 51 | c.SchemaKeys["repository"] = fmt.Sprintf("%s,bucket=%v", pkgPath, bucket) 52 | setupAndSaveConfig(c) 53 | } 54 | -------------------------------------------------------------------------------- /cmd/reflowbootstrap/install.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # install.bash is a script to build a reflow bootstrap binary and push to a public S3 bucket. 4 | # To build and release a new version, update VERSION below and run the script. 5 | # Make sure to update the latest path in: go/src/github.com/grailbio/reflow/cmd/reflow/main.go 6 | # 7 | # Reflow uses this binary as a bootstrap on reflowlet instances. 8 | # 9 | set -e 10 | set -x 11 | 12 | VERSION=reflowbootstrap0.4 13 | 14 | GOOS=linux GOARCH=amd64 go build -o /tmp/$VERSION . 15 | cloudkey ti-apps/admin aws s3 cp --acl public-read /tmp/$VERSION s3://grail-public-bin/linux/amd64/$VERSION 16 | -------------------------------------------------------------------------------- /cmd/reflowbootstrap/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "log" 9 | "os" 10 | 11 | "github.com/grailbio/reflow/bootstrap" 12 | ) 13 | 14 | func main() { 15 | if err := bootstrap.Flags(bootstrap.DefaultSchemaKeys).Parse(os.Args[1:]); err != nil { 16 | log.Fatal(err) 17 | } 18 | bootstrap.RunServer(bootstrap.DefaultSchema, bootstrap.DefaultSchemaKeys) 19 | } 20 | -------------------------------------------------------------------------------- /config.md: -------------------------------------------------------------------------------- 1 | Reflow Configuration 2 | 3 | Configuration in reflow is implemented using the infra (github.com/grailbio/infra) package. 4 | 5 | Infra configuration expects: 6 | - a schema that defines the infrastructure schema and 7 | - schema keys that specifies the providers that satisfy the schema. 8 | 9 | Each provider can choose to implement zero or more of the following methods, based on what it needs to do. 10 | A provider can depend on other providers to initialize or setup itself as long as there are no cyclic dependencies. 11 | - `Init(p1 Provider1, p2 Provider2, ...) error` 12 | - `Setup(pa ProviderA, pb Providerb, ...) error` 13 | - `InstanceConfig() interface{}` 14 | - `Config() interface{}` 15 | - `Version() int` 16 | 17 | -------------------------------------------------------------------------------- /constants.go: -------------------------------------------------------------------------------- 1 | package reflow 2 | 3 | const ( 4 | 5 | // ReflowletProcessMemoryReservationPct is the amount of memory that's reserved for the reflowlet process, 6 | // currently set to 5%. The EC2 overhead and this memory reservation for the reflowlet process, 7 | // will both be accounted for when we do per-instance verification (using `reflow ec2verify`). 8 | // NOTE: Changing this will warrant re-verification of all instance types. 9 | ReflowletProcessMemoryReservationPct = 0.05 10 | ) 11 | 12 | // GetMaxResourcesMemoryBufferFactor is the buffer factor to compute the max memory resources, 13 | // when determining if a given resource requirement is within the threshold of resources an alloc can provide. 14 | // 15 | // Since the max amount of memory available on an alloc is computed based on ReflowletProcessMemoryReservationPct, 16 | // we simply invert that computation to determine the max amount of actual memory likely available on a reflowlet. 17 | func GetMaxResourcesMemoryBufferFactor() float64 { 18 | return 1.0 / (1.0 - ReflowletProcessMemoryReservationPct) 19 | } 20 | -------------------------------------------------------------------------------- /context/context.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package context 6 | 7 | type key int 8 | 9 | const ( 10 | TracerKey key = 0 11 | SpanKey key = 1 12 | 13 | MetricsClientKey key = 2 14 | ) 15 | -------------------------------------------------------------------------------- /digester.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package reflow 6 | 7 | import ( 8 | "crypto" 9 | _ "crypto/sha256" 10 | 11 | "github.com/grailbio/base/digest" 12 | ) 13 | 14 | var Digester = digest.Digester(crypto.SHA256) 15 | 16 | // StringDigest holds any string and its digest. 17 | type StringDigest struct { 18 | s string 19 | d digest.Digest 20 | } 21 | 22 | // NewStringDigest creates a StringDigest based on the given string. 23 | func NewStringDigest(s string) StringDigest { 24 | return StringDigest{ 25 | s: s, 26 | d: Digester.FromString(s), 27 | } 28 | } 29 | 30 | // String returns the underlying string. 31 | func (i StringDigest) String() string { 32 | return i.s 33 | } 34 | 35 | // Digest returns the digest of the underlying string. 36 | func (i StringDigest) Digest() digest.Digest { 37 | return i.d 38 | } 39 | 40 | // IsValid returns whether this StringDigest is valid (ie, the underlying string is non-empty). 41 | func (i StringDigest) IsValid() bool { 42 | return i.s != "" 43 | } 44 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package reflow implements the core data structures and (abstract) runtime 6 | // for Reflow. 7 | // 8 | // Reflow is a system for distributed program execution. The programs are described 9 | // by Flows, which are an abstract specification of the program's execution. Each Flow 10 | // node can take any number of other Flows as dependent inputs and perform some 11 | // (local) execution over these inputs in order to compute some output value. 12 | // 13 | // Reflow supports a limited form of dynamic dependencies: a Flow may evaluate to 14 | // a list of values, each of which may be executed independently. This mechanism 15 | // also provides parallelism. 16 | // 17 | // The system orchestrates Flow execution by evaluating the flow in the manner of an 18 | // abstract syntax tree; see Eval for more details. 19 | package reflow 20 | -------------------------------------------------------------------------------- /doc/1000align/1000align.rf: -------------------------------------------------------------------------------- 1 | // This is the 1000align module. It aligns a single phase-3 sample 2 | // from the 1000genomes project. 3 | 4 | param ( 5 | // sample is the name of the 1000genomes phase 3 sample 6 | sample string 7 | 8 | // out is the target of the output merged BAM file 9 | out string 10 | ) 11 | 12 | // g1kv37 is the 1000genomes version of hg19. 13 | val g1kv37 = file("s3://1000genomes/technical/reference/human_g1k_v37.fasta.gz") 14 | 15 | // g1kv37Indexed is the BWA-indexed version of g1kv37. 16 | val g1kv37Indexed = exec(image := "biocontainers/bwa:v0.7.15_cv3", mem := 6*GiB, cpu := 1) (out dir) {" 17 | # Ignore failures here. The file from 1000genomes has a trailer 18 | # that isn't recognized by gunzip. (This is not recommended practice!) 19 | gunzip -c {{g1kv37}} > {{out}}/g1k_v37.fa || true 20 | cd {{out}} 21 | bwa index -a bwtsw g1k_v37.fa 22 | "} 23 | 24 | val align = make("./align.rf", 25 | sequencingCenter := "1000genomes", 26 | platform := "Illumina", 27 | model := "unknown", 28 | threads := 32, 29 | sample, 30 | alignmentGenome := g1kv37Indexed, 31 | // 1000genomes uses a non-standard grouping pattern, e.g., 32 | // SRR062640_1.filt.fastq.gz, SRR062640_2.filt.fastq.gz 33 | groupPattern := "(.*)_[12].filt.fastq.gz$", 34 | // We don't really know metadata, so always return empty. 35 | fastqMeta := func(name string) => ("unknown", "unknown")) 36 | 37 | val fastqs = dir("s3://1000genomes/phase3/data/"+sample+"/sequence_read/") 38 | val bam = make("./bam.rf") 39 | val merged = bam.Merge(align.AlignPairs(fastqs)) 40 | 41 | val files = make("$/files") 42 | 43 | // Align a sample from the 1000genomes phase 3 dataset into a single, 44 | // merged BAM file. The module is parameterized on the sample name 45 | // (e.g., HG00103), and the output (e.g., an s3 path). 46 | @requires(cpu := 32, mem := 128*GiB, disk := 80*GiB) 47 | val Main = files.Copy(merged, out) 48 | -------------------------------------------------------------------------------- /doc/1000align/bam.rf: -------------------------------------------------------------------------------- 1 | 2 | param ( 3 | // threads governs how many threads to use by default in computations. 4 | threads = 16 5 | ) 6 | 7 | val dirs = make("$/dirs") 8 | 9 | val samtools = "biocontainers/samtools:v1.7.0_cv2" 10 | val sambamba = "??" 11 | 12 | // Index indexes a BAM file using Samtools. 13 | func Index(bam file) file = 14 | exec(image := samtools, cpu := threads) (indexed file) {" 15 | samtools index -@ {{threads*2}} {{bam}} {{indexed}} 16 | "} 17 | 18 | // Sort sorts a BAM file using Samtools 19 | func Sort(bam file) file = 20 | exec(image := samtools, cpu := threads, mem := GiB) (sorted file) {" 21 | samtools sort --threads {{threads*4}} \ 22 | -o {{sorted}} {{bam}} 23 | "} 24 | 25 | // ShardBAM splits a BAM file according to genome regions BED file. 26 | func Shard(bam, regions file) dir = 27 | exec(image := samtools, cpu := threads) (shards dir) {" 28 | parallel -a {{regions}} \ 29 | --colsep '\t' -j {{threads}} --delay 1 \ 30 | 'samtools view -bh {{bam}} {1}:{2}-{3} 31 | > {{shards}}/{1}_{2}_{3}.bam)' 32 | "} 33 | 34 | // Merge merges a list of BAM files into a single BAM file. 35 | func Merge(bams [file]) file = 36 | if len(bams) == 1 { 37 | val [bam] = bams 38 | bam 39 | } else { 40 | exec(image := samtools, cpu := threads) (merged file) {" 41 | samtools merge -@{{threads*2}} {{merged}} {{bams}} 42 | "} 43 | } 44 | 45 | func MarkDuplicates(bam file) (bam, bai file) = { 46 | val out = exec(image := sambamba, cpu := threads, mem := 32*GiB) (out dir) {" 47 | sambamba_v0.6.5 markdup \ 48 | --tmpdir $tmp \ 49 | -t {{threads*2}} \ 50 | --sort-buffer-size 24000 \ 51 | --overflow-list-size 10000000 \ 52 | {{bam}} {{out}}/dupmarked.bam 53 | "} 54 | 55 | val (dupmarked, _) = dirs.Pick(out, "dupmarked.bam") 56 | val (index, _) = dirs.Pick(out, "dupmarked.bam.bai") 57 | (dupmarked, index) 58 | } 59 | 60 | func FilterDedupAndMapQ(bam file, mapq int) (bam file) = 61 | exec(image := sambamba, cpu := threads) (filtered file) {" 62 | sambamba_v0.6.5 view -F "mapping_quality >= {{mapq}} and not duplicate" \ 63 | -t {{threads*2}} \ 64 | -f bam {{bam}} > {{filtered}} 65 | "} 66 | -------------------------------------------------------------------------------- /doc/1000align/fastq.rf: -------------------------------------------------------------------------------- 1 | val dirs = make("$/dirs") 2 | val image = "ubuntu" 3 | 4 | // Split splits a fastq file into multiple FASTQ files with 5 | // a given number of reads. 6 | func Split(fastq file, reads int) [file] = { 7 | splits := exec(image) (splits dir) {" 8 | zcat {{fastq}} | split -l {{reads*4}} - \ 9 | {{splits}}/split_ --filter='gzip > $FILE.gz' 10 | "} 11 | dirs.Files(splits) 12 | } 13 | 14 | -------------------------------------------------------------------------------- /doc/images/annotated_trace_overview.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/doc/images/annotated_trace_overview.jpeg -------------------------------------------------------------------------------- /doc/images/selection_tool.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/doc/images/selection_tool.jpeg -------------------------------------------------------------------------------- /doc/images/timing_tool.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/doc/images/timing_tool.jpeg -------------------------------------------------------------------------------- /ec2authenticator/ec2authenticator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2authenticator 6 | 7 | import "testing" 8 | 9 | func TestRegexp(t *testing.T) { 10 | for _, ok := range []string{ 11 | "012345678910.dkr.ecr.us-west-2.amazonaws.com/ubuntu", 12 | "012345678910.dkr.ecr.us-west-2.amazonaws.com/amazonlinux", 13 | "012345678910.dkr.ecr.us-west-2.amazonaws.com/windows_sample_app", 14 | "012345678910.dkr.ecr.us-west-2.amazonaws.com/wgs:v2", 15 | } { 16 | if !ecrURI.MatchString(ok) { 17 | t.Errorf("expected match for %s", ok) 18 | } 19 | } 20 | 21 | for _, notok := range []string{ 22 | "ubuntu", 23 | "alpine/linux", 24 | "monkey.org/docker/blah", 25 | "xyz012345678910.dkr.ecr.us-west-2.amazonaws.com/amazonlinux", 26 | } { 27 | if ecrURI.MatchString(notok) { 28 | t.Errorf("did not expect match for %s", notok) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ec2cluster/amis.go: -------------------------------------------------------------------------------- 1 | // THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT. 2 | // Generated from URL: https://stable.release.flatcar-linux.net/amd64-usr/current/flatcar_production_ami_all.json 3 | // FLATCAR_BUILD=3510 4 | // FLATCAR_BRANCH=2 5 | // FLATCAR_PATCH=8 6 | // FLATCAR_VERSION=3510.2.8 7 | // FLATCAR_VERSION_ID=3510.2.8 8 | // FLATCAR_BUILD_ID="2023-09-19-1914" 9 | // FLATCAR_SDK_VERSION=3510.0.0 10 | 11 | package ec2cluster 12 | 13 | import ( 14 | "fmt" 15 | 16 | "github.com/aws/aws-sdk-go/aws/session" 17 | ) 18 | 19 | var flatcarAmiByRegion = map[string]string{ 20 | "af-south-1": "ami-018d9b9d7151ab6c8", 21 | "ap-east-1": "ami-0a20232670e0772af", 22 | "ap-northeast-1": "ami-0233879a9dd92df11", 23 | "ap-northeast-2": "ami-01bff27d84a7a7700", 24 | "ap-south-1": "ami-0cc2acda59cd3af78", 25 | "ap-southeast-1": "ami-0c42cc418ecd92604", 26 | "ap-southeast-2": "ami-01691d179b95f6f8c", 27 | "ap-southeast-3": "ami-0dfb599c8fc910601", 28 | "ca-central-1": "ami-09dc356370f64513d", 29 | "eu-central-1": "ami-068829f5bf9333501", 30 | "eu-north-1": "ami-0a560c9c18e52fe3a", 31 | "eu-south-1": "ami-0b069d1752bac2e31", 32 | "eu-west-1": "ami-0e28f7527ce1dd0d1", 33 | "eu-west-2": "ami-03f8f46c0d8a86c5c", 34 | "eu-west-3": "ami-0328a3887119e6dd3", 35 | "me-south-1": "ami-0f34884e3fbf09f49", 36 | "sa-east-1": "ami-0151f43dfd95a3865", 37 | "us-east-1": "ami-0604fd4b5a31a9e9f", 38 | "us-east-2": "ami-0a804cb708f395c8a", 39 | "us-west-1": "ami-0ba006ff9272db530", 40 | "us-west-2": "ami-0c3488073a14ab0af", 41 | } 42 | 43 | // GetAMI gets the AMI ID for the AWS region derived from the given AWS session. 44 | func GetAMI(sess *session.Session) (string, error) { 45 | region := *sess.Config.Region 46 | ami, ok := flatcarAmiByRegion[region] 47 | if !ok { 48 | return "", fmt.Errorf("no AMI defined for region (derived from AWS session): %s", region) 49 | } 50 | return ami, nil 51 | } 52 | -------------------------------------------------------------------------------- /ec2cluster/cloudconfig_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2cluster 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | ) 11 | 12 | func TestCloudconfig(t *testing.T) { 13 | var c cloudConfig 14 | c.CoreOS.Update.RebootStrategy = "off" 15 | c.AppendFile(CloudFile{Path: "/tmp/x", Permissions: "0644", Owner: "root", Content: "a test file"}) 16 | c.AppendUnit(CloudUnit{"reflowlet", "command", true, "unit content"}) 17 | var d cloudConfig 18 | d.AppendUnit(CloudUnit{"xxx", "xxxcommand", false, "xxx content"}) 19 | d.AppendFile(CloudFile{Path: "/tmp/myfile", Permissions: "0644", Owner: "root", Content: "another test file"}) 20 | c.Merge(&d) 21 | out, err := c.Marshal() 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | if got, want := out, []byte(`#cloud-config 26 | write_files: 27 | - path: /tmp/x 28 | permissions: "0644" 29 | owner: root 30 | content: a test file 31 | - path: /tmp/myfile 32 | permissions: "0644" 33 | owner: root 34 | content: another test file 35 | coreos: 36 | update: 37 | reboot-strategy: "off" 38 | units: 39 | - name: reflowlet 40 | command: command 41 | enable: true 42 | content: unit content 43 | - name: xxx 44 | command: xxxcommand 45 | content: xxx content 46 | `); !bytes.Equal(got, want) { 47 | t.Errorf("got %q, want %q", got, want) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /ec2cluster/ecr_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2cluster 6 | 7 | import ( 8 | "github.com/aws/aws-sdk-go/service/ecr" 9 | "github.com/aws/aws-sdk-go/service/ecr/ecriface" 10 | ) 11 | 12 | type mockECRClient struct { 13 | ecriface.ECRAPI 14 | 15 | Err error // If set, calls to DescribeImages() will return this error. 16 | } 17 | 18 | // DescribeImages the call on DescribeInstancesInputs 19 | // and returns an empty result. 20 | func (e *mockECRClient) DescribeImages(input *ecr.DescribeImagesInput) (*ecr.DescribeImagesOutput, error) { 21 | if e.Err != nil { 22 | return nil, e.Err 23 | } 24 | return new(ecr.DescribeImagesOutput), nil 25 | } 26 | -------------------------------------------------------------------------------- /ec2cluster/spotaz_test.go: -------------------------------------------------------------------------------- 1 | package ec2cluster 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/ec2" 8 | ) 9 | 10 | func TestComputeAzSubnetMap(t *testing.T) { 11 | client := &mockEC2Client{descSubnetOut: &ec2.DescribeSubnetsOutput{ 12 | Subnets: []*ec2.Subnet{ 13 | {AvailabilityZone: aws.String("abc"), SubnetId: aws.String("abc-id1")}, 14 | {AvailabilityZone: aws.String("def"), SubnetId: aws.String("def-id1")}, 15 | }, 16 | }} 17 | azNameToSubnetOnce.Reset() 18 | if err := computeAzSubnetMap(client, []string{ /* doesn't matter for test */ }, nil); err != nil { 19 | t.Fatal(err) 20 | } 21 | if got, want := subnetForAZ("abc"), "abc-id1"; got != want { 22 | t.Errorf("got %s, want %s", got, want) 23 | } 24 | if got, want := subnetForAZ("def"), "def-id1"; got != want { 25 | t.Errorf("got %s, want %s", got, want) 26 | } 27 | 28 | client = &mockEC2Client{descSubnetOut: &ec2.DescribeSubnetsOutput{ 29 | Subnets: []*ec2.Subnet{ 30 | {AvailabilityZone: aws.String("abc"), SubnetId: aws.String("abc-id1")}, 31 | {AvailabilityZone: aws.String("def"), SubnetId: aws.String("def-id1")}, 32 | {AvailabilityZone: aws.String("abc"), SubnetId: aws.String("abc-id2")}, 33 | }, 34 | }} 35 | 36 | azNameToSubnetOnce.Reset() 37 | if err := computeAzSubnetMap(client, []string{ /* doesn't matter for test */ }, nil); err != nil { 38 | t.Fatal(err) 39 | } 40 | if got, want := subnetForAZ("abc"), "abc-id2"; got != want { 41 | t.Errorf("got %s, want %s", got, want) 42 | } 43 | if got, want := subnetForAZ("def"), "def-id1"; got != want { 44 | t.Errorf("got %s, want %s", got, want) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /ec2cluster/test/fakecluster.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | 7 | "github.com/grailbio/base/data" 8 | "github.com/grailbio/infra" 9 | "github.com/grailbio/reflow" 10 | "github.com/grailbio/reflow/ec2cluster" 11 | "github.com/grailbio/reflow/pool" 12 | ) 13 | 14 | func init() { 15 | infra.Register("fakecluster", new(Cluster)) 16 | } 17 | 18 | // Cluster is a fake cluster infra provider and should be used only in tests. 19 | type Cluster struct { 20 | *ec2cluster.Cluster 21 | } 22 | 23 | // Init implements infra.Provider. 24 | func (c *Cluster) Init() error { 25 | c.Cluster = &ec2cluster.Cluster{} 26 | return nil 27 | } 28 | 29 | func (c *Cluster) CanAllocate(r reflow.Resources) (bool, error) { 30 | return true, nil 31 | } 32 | 33 | var maxResources = reflow.Resources{"cpu": 16, "mem": 64 * float64(data.GiB)} 34 | 35 | // MaxAlloc returns the max resources which can be obtained in a single alloc from this cluster. 36 | func (c *Cluster) MaxAlloc() reflow.Resources { 37 | return maxResources 38 | } 39 | 40 | // Allocate is a no op. 41 | func (c *Cluster) Allocate(ctx context.Context, req reflow.Requirements, labels pool.Labels) (pool.Alloc, error) { 42 | return nil, nil 43 | } 44 | 45 | // GetName implements runner.Cluster 46 | func (c *Cluster) GetName() string { return "fakecluster" } 47 | 48 | // Flags implements infra.Provider 49 | func (r *Cluster) Flags(flags *flag.FlagSet) { 50 | var ignoredString string 51 | var ignoredBool bool 52 | flags.StringVar(&ignoredString, "instancetypes", "", "(IGNORED) instance types") 53 | flags.BoolVar(&ignoredBool, "usespot", false, "(IGNORED) use spot or ondemand") 54 | } 55 | -------------------------------------------------------------------------------- /ec2cluster/tmpl_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2cluster 6 | 7 | import "testing" 8 | 9 | func TestTmpl(t *testing.T) { 10 | got := tmpl(` 11 | x, y, {{.z}} 12 | blah 13 | bloop 14 | `, args{"z": 1}) 15 | want := `x, y, 1 16 | blah 17 | bloop` 18 | if got != want { 19 | t.Errorf("got %q, want %q", got, want) 20 | } 21 | } 22 | 23 | func TestBadTmpl(t *testing.T) { 24 | v := recoverPanic(func() { 25 | tmpl(` 26 | x, y 27 | blah`, nil) 28 | }) 29 | if v == nil { 30 | t.Fatal("expected panic") 31 | } 32 | if got, want := v.(string), `nonspace prefix in "\t\t\tblah"`; got != want { 33 | t.Errorf("got %v, want %v", got, want) 34 | } 35 | } 36 | 37 | func recoverPanic(f func()) (v interface{}) { 38 | defer func() { 39 | v = recover() 40 | }() 41 | f() 42 | return 43 | } 44 | -------------------------------------------------------------------------------- /ec2cluster/volume/ec2volume_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package volume 6 | 7 | import ( 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/request" 10 | "github.com/aws/aws-sdk-go/service/ec2" 11 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 12 | ) 13 | 14 | type mockEC2Client struct { 15 | ec2iface.EC2API 16 | descVolsFn func(volIds []string) (*ec2.DescribeVolumesOutput, error) 17 | descVolsModsFn func(volIds []string) (*ec2.DescribeVolumesModificationsOutput, error) 18 | modVolsFn func(volId string) (*ec2.ModifyVolumeOutput, error) 19 | 20 | nModVols int 21 | } 22 | 23 | func (e *mockEC2Client) DescribeVolumesWithContext(_ aws.Context, input *ec2.DescribeVolumesInput, _ ...request.Option) (*ec2.DescribeVolumesOutput, error) { 24 | ids := make([]string, 0, len(input.VolumeIds)) 25 | for _, id := range input.VolumeIds { 26 | ids = append(ids, aws.StringValue(id)) 27 | } 28 | return e.descVolsFn(ids) 29 | } 30 | 31 | func (e *mockEC2Client) DescribeVolumesModificationsWithContext(_ aws.Context, input *ec2.DescribeVolumesModificationsInput, _ ...request.Option) (*ec2.DescribeVolumesModificationsOutput, error) { 32 | ids := make([]string, 0, len(input.VolumeIds)) 33 | for _, id := range input.VolumeIds { 34 | ids = append(ids, aws.StringValue(id)) 35 | } 36 | return e.descVolsModsFn(ids) 37 | } 38 | 39 | func (e *mockEC2Client) ModifyVolumeWithContext(_ aws.Context, input *ec2.ModifyVolumeInput, _ ...request.Option) (*ec2.ModifyVolumeOutput, error) { 40 | e.nModVols++ 41 | return e.modVolsFn(aws.StringValue(input.VolumeId)) 42 | } 43 | -------------------------------------------------------------------------------- /executor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package reflow_test 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | 11 | "docker.io/go-docker/api/types" 12 | "github.com/grailbio/reflow" 13 | ) 14 | 15 | func TestRuntime(t *testing.T) { 16 | var ( 17 | e = new(reflow.ExecInspect) 18 | b = new(types.ContainerJSONBase) 19 | d = new(types.ContainerJSON) 20 | s = new(types.ContainerState) 21 | ) 22 | b.State = s 23 | d.ContainerJSONBase = b 24 | e.Docker = *d 25 | 26 | for _, tt := range []struct { 27 | sd, ed string 28 | want time.Duration 29 | }{ 30 | // End is around 0.01s before start. In this case, the runtime should be 0s. 31 | {"2019-05-30T22:09:34.945074271Z", "2019-05-30T22:09:34.910391085Z", time.Duration(0)}, 32 | // End is 10 seconds after start, rounded to the nearest second. 33 | {"2019-05-30T22:09:34.945074271Z", "2019-05-30T22:09:44.910391085Z", 10 * time.Second}, 34 | // End is 10 seconds after start, rounded to the nearest second. End is also missing a digit (8 decimals instead of 9). 35 | {"2019-05-30T22:09:34.945074271Z", "2019-05-30T22:09:44.91039105Z", 10 * time.Second}, 36 | } { 37 | s.StartedAt, s.FinishedAt = tt.sd, tt.ed 38 | if got, want := e.Runtime().Round(time.Second), tt.want; got != want { 39 | t.Errorf("got %v, want %v", got, want) 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /fileset.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package reflow; 4 | 5 | option go_package = "github.com/grailbio/reflow"; 6 | 7 | message Timestamp { 8 | int64 Seconds = 1; 9 | int64 Nanos = 2; 10 | } 11 | 12 | message FileP { 13 | string id = 1; 14 | int64 size = 2; 15 | string source = 3; 16 | string etag = 4; 17 | Timestamp lastModified = 5; 18 | string contentHash = 6; 19 | } 20 | 21 | message FilePart { 22 | int32 id = 1; 23 | FileP file = 2; 24 | int32 assertionsGroupId = 3; 25 | } 26 | 27 | message FileMappingPart { 28 | int32 depth = 1; 29 | int32 index = 2; 30 | string key = 3; 31 | int32 fileId = 4; 32 | } 33 | 34 | message BlobProperties { 35 | string etag = 1; 36 | string lastModified = 2; 37 | string size = 3; 38 | } 39 | 40 | message AssertionsKeyPart { 41 | int32 id = 1; 42 | string subject = 2; 43 | oneof properties { 44 | BlobProperties bp = 3; 45 | } 46 | } 47 | 48 | message AssertionsGroupPart { 49 | int32 id = 1; 50 | repeated int32 keyIds = 2 [packed = true]; 51 | } 52 | 53 | message FilesetPart { 54 | oneof part { 55 | FilePart fp = 1; 56 | FileMappingPart fmp = 2; 57 | AssertionsKeyPart akp = 3; 58 | AssertionsGroupPart agp = 4; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /filesetjson.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package reflow 6 | 7 | import ( 8 | "fmt" 9 | "runtime" 10 | "sync" 11 | 12 | "github.com/grailbio/base/limiter" 13 | ) 14 | 15 | type FilesetLimiter struct { 16 | *limiter.Limiter 17 | n int 18 | } 19 | 20 | func newFilesetLimiter(limit int) *FilesetLimiter { 21 | l := &FilesetLimiter{limiter.New(), limit} 22 | l.Release(limit) 23 | return l 24 | } 25 | 26 | func (l *FilesetLimiter) Limit() int { 27 | return l.n 28 | } 29 | 30 | var ( 31 | // filesetOpLimiter is the number concurrent Fileset marshal/unmarshaling operations. 32 | // When large number of these operations are done concurrently (especially if the Filesets are large), 33 | // marshaling/unmarshaling too many of them can cause OOMs, 34 | // TODO(swami): Better solution is to use a more optimized Fileset format (either JSON or other). 35 | filesetOpLimiter *FilesetLimiter 36 | limiterOnce sync.Once 37 | filesetOpLimit = runtime.NumCPU() 38 | ) 39 | 40 | // SetFilesetOpConcurrencyLimit sets the limit of concurrent fileset operations. 41 | // For a successful reset of the limit, this should be called before a call is made 42 | // to GetFilesetOpLimiter, ie before any reflow evaluations have started, otherwise this will panic. 43 | func SetFilesetOpConcurrencyLimit(limit int) { 44 | if filesetOpLimiter != nil { 45 | panic(fmt.Sprintf("cannot reset marshal limit to %d (already set at %d)", limit, filesetOpLimiter.Limit())) 46 | } 47 | if limit <= 0 { 48 | return 49 | } 50 | filesetOpLimit = limit 51 | } 52 | 53 | func GetFilesetOpLimiter() *FilesetLimiter { 54 | limiterOnce.Do(func() { 55 | filesetOpLimiter = newFilesetLimiter(filesetOpLimit) 56 | }) 57 | return filesetOpLimiter 58 | } 59 | -------------------------------------------------------------------------------- /flow/assoc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow_test 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/grailbio/base/digest" 11 | "github.com/grailbio/reflow/assoc" 12 | "github.com/grailbio/reflow/errors" 13 | ) 14 | 15 | type waitAssoc struct { 16 | assoc.Assoc 17 | tick chan struct{} 18 | } 19 | 20 | func newWaitAssoc() *waitAssoc { 21 | return &waitAssoc{tick: make(chan struct{})} 22 | } 23 | 24 | func (w *waitAssoc) Tick() { 25 | w.tick <- struct{}{} 26 | } 27 | 28 | func (w *waitAssoc) Get(ctx context.Context, kind assoc.Kind, k digest.Digest) (kexp, v digest.Digest, err error) { 29 | <-w.tick 30 | err = errors.E(errors.NotExist) 31 | return 32 | } 33 | 34 | func (w *waitAssoc) BatchGet(ctx context.Context, batch assoc.Batch) (err error) { 35 | for i := 0; i < len(batch); i++ { 36 | <-w.tick 37 | } 38 | err = errors.E(errors.NotExist) 39 | return 40 | } 41 | -------------------------------------------------------------------------------- /flow/context.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import "context" 8 | 9 | var contextKey struct{} 10 | 11 | // Context is a context.Context that is used for background 12 | // operations within reflow. In addition to providing a common 13 | // background context for operations, it also carries a WaitGroup, so 14 | // that the caller can wait for background operation completion. 15 | type Context struct { 16 | context.Context 17 | wg WaitGroup 18 | } 19 | 20 | // Background returns the Context associated with the given / parent 21 | // context.Context. If there is no associated context, it returns a 22 | // fresh Context without an affiliated WaitGroup. 23 | func Background(ctx context.Context) Context { 24 | if ctx, ok := ctx.Value(&contextKey).(Context); ok { 25 | ctx.wg.Add(1) 26 | return ctx 27 | } 28 | return Context{context.Background(), nopWaitGroup{}} 29 | } 30 | 31 | // Complete should be called when the operation is complete. 32 | func (c Context) Complete() { 33 | c.wg.Done() 34 | } 35 | 36 | // WaitGroup defines a subset of sync.WaitGroup's interface 37 | // for use with Context. 38 | type WaitGroup interface { 39 | Add(int) 40 | Done() 41 | } 42 | 43 | type nopWaitGroup struct{} 44 | 45 | func (nopWaitGroup) Add(int) {} 46 | func (nopWaitGroup) Done() {} 47 | 48 | // WithBackground returns a new context.Context with an affiliated 49 | // Context, accessible via Background. The background context may be 50 | // canceled with the returned cancellation function. The supplied 51 | // WaitGroup is used to inform the caller of pending background 52 | // operations: wg.Add(1) is called for each call to Background; 53 | // wg.Done is called when the context returned from Background is 54 | // disposed of through (Context).Complete. 55 | func WithBackground(ctx context.Context, wg WaitGroup) (context.Context, context.CancelFunc) { 56 | bg, cancel := context.WithCancel(context.Background()) 57 | return context.WithValue(ctx, &contextKey, Context{bg, wg}), cancel 58 | } 59 | -------------------------------------------------------------------------------- /flow/context_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow_test 6 | 7 | import ( 8 | "context" 9 | "sync" 10 | "testing" 11 | 12 | "github.com/grailbio/reflow/flow" 13 | ) 14 | 15 | type nopWaitGroup struct{} 16 | 17 | func (nopWaitGroup) Add(int) {} 18 | func (nopWaitGroup) Done() {} 19 | 20 | func TestContext(t *testing.T) { 21 | ctx := context.Background() 22 | _ = flow.Background(ctx) // nil ok. 23 | 24 | var wg1, wg2 sync.WaitGroup 25 | ctx1, _ := flow.WithBackground(ctx, &wg1) 26 | ctx2, _ := flow.WithBackground(ctx, &wg2) 27 | 28 | const N = 100 29 | var ctxs1, ctxs2 [N]flow.Context 30 | for i := 0; i < N; i++ { 31 | ctxs1[i], ctxs2[i] = flow.Background(ctx1), flow.Background(ctx2) 32 | } 33 | done := make(chan bool) 34 | for i := 0; i < N; i++ { 35 | go func(i int) { 36 | ctxs1[i].Complete() 37 | wg2.Wait() 38 | done <- true 39 | }(i) 40 | } 41 | wg1.Wait() 42 | for i := 0; i < N; i++ { 43 | select { 44 | case <-done: 45 | t.Fatal("WaitGroup released too soon") 46 | default: 47 | } 48 | ctxs2[i].Complete() 49 | } 50 | for i := 0; i < N; i++ { 51 | <-done 52 | } 53 | } 54 | 55 | func TestContextCancel(t *testing.T) { 56 | ctx, cancelParent := context.WithCancel(context.Background()) 57 | ctx, cancel := flow.WithBackground(ctx, nopWaitGroup{}) 58 | bgctx := flow.Background(ctx) 59 | 60 | if err := ctx.Err(); err != nil { 61 | t.Error(err) 62 | } 63 | cancelParent() 64 | if got, want := ctx.Err(), context.Canceled; got != want { 65 | t.Errorf("got %v, want %v", got, want) 66 | } 67 | if err := bgctx.Err(); err != nil { 68 | t.Error(err) 69 | } 70 | cancel() 71 | if got, want := bgctx.Err(), context.Canceled; got != want { 72 | t.Errorf("got %v, want %v", got, want) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /flow/export_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/grailbio/base/digest" 11 | "github.com/grailbio/reflow" 12 | ) 13 | 14 | var ( 15 | MemMultiplier = memMultiplier 16 | OomRetryMaxExecMemory = oomRetryMaxExecMemory 17 | ) 18 | 19 | func PhysicalDigests(f *Flow) []digest.Digest { 20 | return f.physicalDigests() 21 | } 22 | 23 | func DepAssertions(f *Flow) []*reflow.Assertions { 24 | return f.depAssertions() 25 | } 26 | 27 | type AssertionsBatchCache = assertionsBatchCache 28 | 29 | func NewAssertionsBatchCache(e *Eval) *AssertionsBatchCache { 30 | return e.newAssertionsBatchCache() 31 | } 32 | 33 | func RefreshAssertions(ctx context.Context, e *Eval, a []*reflow.Assertions, cache *AssertionsBatchCache) ([]*reflow.Assertions, error) { 34 | return e.refreshAssertions(ctx, a, cache) 35 | } 36 | 37 | func OomAdjust(specified, used reflow.Resources) reflow.Resources { 38 | return oomAdjust(specified, used) 39 | } 40 | 41 | // FindFlowCopy finds the copy of a given flow in the flow graph maintained by the Eval. 42 | // Useful for performing assertions on Flow properties post-evaluation. 43 | func (e *Eval) FindFlowCopy(f *Flow) *Flow { 44 | v := e.root.Visitor() 45 | for v.Walk() { 46 | if f.Digest() == v.Digest() { 47 | return v.Flow 48 | } 49 | v.Visit() 50 | } 51 | return nil 52 | } 53 | 54 | func (e *EvalConfig) CapMemory(r reflow.Resources) (rr reflow.Resources, capped bool, err error) { 55 | return e.capMemory(r) 56 | } 57 | -------------------------------------------------------------------------------- /flow/fileset.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import ( 8 | "github.com/grailbio/reflow" 9 | "github.com/grailbio/reflow/values" 10 | ) 11 | 12 | // filesetFlow returns the Flow which evaluates to the constant Value v. 13 | func filesetFlow(v reflow.Fileset) *Flow { 14 | return &Flow{Op: Val, Value: values.T(v), State: Done} 15 | } 16 | -------------------------------------------------------------------------------- /flow/mutation_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Mutation"; DO NOT EDIT. 2 | 3 | package flow 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[Invalid-0] 12 | _ = x[Cached-1] 13 | _ = x[Refresh-2] 14 | _ = x[MustIntern-3] 15 | _ = x[NoStatus-4] 16 | _ = x[Propagate-5] 17 | } 18 | 19 | const _Mutation_name = "InvalidCachedRefreshMustInternNoStatusPropagate" 20 | 21 | var _Mutation_index = [...]uint8{0, 7, 13, 20, 30, 38, 47} 22 | 23 | func (i Mutation) String() string { 24 | if i < 0 || i >= Mutation(len(_Mutation_index)-1) { 25 | return "Mutation(" + strconv.FormatInt(int64(i), 10) + ")" 26 | } 27 | return _Mutation_name[_Mutation_index[i]:_Mutation_index[i+1]] 28 | } 29 | -------------------------------------------------------------------------------- /flow/op_string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Code generated by "stringer -type=Op"; DO NOT EDIT 6 | 7 | package flow 8 | 9 | import "fmt" 10 | 11 | const _Op_name = "OpExecOpInternOpExternOpGroupbyOpMapOpCollectOpMergeOpValOpPullupOpKOpCoerceOpRequirementsmaxOp" 12 | 13 | var _Op_index = [...]uint8{0, 6, 14, 22, 31, 36, 45, 52, 57, 65, 68, 76, 90, 95} 14 | 15 | func (i Op) DigestString() string { 16 | i -= 1 17 | if i < 0 || i >= Op(len(_Op_index)-1) { 18 | return fmt.Sprintf("Op(%d)", i+1) 19 | } 20 | return _Op_name[_Op_index[i]:_Op_index[i+1]] 21 | } 22 | -------------------------------------------------------------------------------- /flow/prefixwriter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | ) 11 | 12 | // prefixWriter is an io.Writer that outputs a prefix before each line. 13 | type prefixWriter struct { 14 | w io.Writer 15 | prefix string 16 | needPrefix bool 17 | } 18 | 19 | func newPrefixWriter(w io.Writer, prefix string) *prefixWriter { 20 | return &prefixWriter{w: w, prefix: prefix, needPrefix: true} 21 | } 22 | 23 | func (w *prefixWriter) Write(p []byte) (n int, err error) { 24 | if w.needPrefix { 25 | if n, err := io.WriteString(w.w, w.prefix); err != nil { 26 | return n, err 27 | } 28 | w.needPrefix = false 29 | } 30 | for { 31 | i := bytes.Index(p, []byte{'\n'}) 32 | switch i { 33 | case len(p) - 1: 34 | w.needPrefix = true 35 | fallthrough 36 | case -1: 37 | return w.w.Write(p) 38 | default: 39 | n, err := w.w.Write(p[:i+1]) 40 | if err != nil { 41 | return n, err 42 | } 43 | _, err = io.WriteString(w.w, w.prefix) 44 | if err != nil { 45 | return n, err 46 | } 47 | p = p[i+1:] 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /flow/prefixwriter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "testing" 11 | ) 12 | 13 | func TestPrefixWriter(t *testing.T) { 14 | var b bytes.Buffer 15 | w := newPrefixWriter(&b, "prefix: ") 16 | io.WriteString(w, "hello") 17 | io.WriteString(w, "\nworld\n\n") 18 | io.WriteString(w, "another\ntest\nthere\n") 19 | if got, want := b.String(), `prefix: hello 20 | prefix: world 21 | prefix: 22 | prefix: another 23 | prefix: test 24 | prefix: there 25 | `; got != want { 26 | t.Errorf("got %q, want %q", got, want) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /flow/state_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=State"; DO NOT EDIT. 2 | 3 | package flow 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[Init-0] 12 | _ = x[NeedLookup-1] 13 | _ = x[Lookup-2] 14 | _ = x[TODO-3] 15 | _ = x[Ready-4] 16 | _ = x[NeedSubmit-5] 17 | _ = x[Running-6] 18 | _ = x[Execing-7] 19 | _ = x[Done-8] 20 | _ = x[Max-9] 21 | } 22 | 23 | const _State_name = "InitNeedLookupLookupTODOReadyNeedSubmitRunningExecingDoneMax" 24 | 25 | var _State_index = [...]uint8{0, 4, 14, 20, 24, 29, 39, 46, 53, 57, 60} 26 | 27 | func (i State) String() string { 28 | if i < 0 || i >= State(len(_State_index)-1) { 29 | return "State(" + strconv.FormatInt(int64(i), 10) + ")" 30 | } 31 | return _State_name[_State_index[i]:_State_index[i+1]] 32 | } 33 | -------------------------------------------------------------------------------- /flow/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import ( 8 | "fmt" 9 | "sort" 10 | ) 11 | 12 | // stats is a simple data structure to keep track of summary 13 | // statistics and histograms. stats stores all samples, 14 | // so it should be used only for small datasets. 15 | type stats struct { 16 | samples []float64 17 | } 18 | 19 | func (s stats) N() int { 20 | return len(s.samples) 21 | } 22 | 23 | func (s *stats) Add(v float64) { 24 | s.samples = append(s.samples, v) 25 | sort.Sort(s) 26 | } 27 | 28 | func (s stats) Mean() float64 { 29 | var total float64 30 | if len(s.samples) == 0 { 31 | return total 32 | } 33 | for _, d := range s.samples { 34 | total += d 35 | } 36 | return total / float64(len(s.samples)) 37 | } 38 | 39 | func (s stats) Percentile(pct int) float64 { 40 | n := len(s.samples) 41 | if n == 0 { 42 | return 0 43 | } 44 | if pct == 100 { 45 | return s.samples[len(s.samples)-1] 46 | } 47 | idx := n * pct / 100 48 | return s.samples[idx] 49 | } 50 | 51 | func (s stats) Summary(format string) string { 52 | return fmt.Sprintf(format+"/"+format+"/"+format, 53 | s.Percentile(0), s.Mean(), s.Percentile(100)) 54 | } 55 | 56 | func (s stats) SummaryScaled(format string, scale float64) string { 57 | return fmt.Sprintf(format+"/"+format+"/"+format, 58 | s.Percentile(0)*scale, s.Mean()*scale, s.Percentile(100)*scale) 59 | } 60 | 61 | func (s stats) Len() int { return len(s.samples) } 62 | func (s stats) Less(i, j int) bool { return s.samples[i] < s.samples[j] } 63 | func (s stats) Swap(i, j int) { s.samples[i], s.samples[j] = s.samples[j], s.samples[i] } 64 | -------------------------------------------------------------------------------- /flow/stats_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import "testing" 8 | 9 | func TestStats(t *testing.T) { 10 | var s stats 11 | if got, want := s.N(), 0; got != want { 12 | t.Fatalf("got %v, want %v", got, want) 13 | } 14 | 15 | s.Add(10) 16 | s.Add(20) 17 | if got, want := s.N(), 2; got != want { 18 | t.Fatalf("got %v, want %v", got, want) 19 | } 20 | if got, want := s.Mean(), float64(15); got != want { 21 | t.Fatalf("got %v, want %v", got, want) 22 | } 23 | expect := []struct { 24 | pct int 25 | v float64 26 | }{ 27 | {1, 10}, 28 | {20, 10}, 29 | {49, 10}, 30 | {51, 20}, 31 | {99, 20}, 32 | } 33 | for _, e := range expect { 34 | if got, want := s.Percentile(e.pct), e.v; got != want { 35 | t.Fatalf("got %v, want %v", got, want) 36 | } 37 | } 38 | if got, want := s.Summary("%.1f"), "10.0/15.0/20.0"; got != want { 39 | t.Fatalf("got %v, want %v", got, want) 40 | } 41 | if got, want := s.SummaryScaled("%.1f", 0.1), "1.0/1.5/2.0"; got != want { 42 | t.Fatalf("got %v, want %v", got, want) 43 | } 44 | if got, want := s.SummaryScaled("%.2f", 0.033), "0.33/0.49/0.66"; got != want { 45 | t.Fatalf("got %v, want %v", got, want) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /flow/strutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import ( 8 | "strings" 9 | "unicode" 10 | "unicode/utf8" 11 | ) 12 | 13 | // abbrev abbreviates string s to a length of max. 14 | func abbrev(s string, max int) string { 15 | if len(s) <= max { 16 | return s 17 | } 18 | a := make([]byte, 0, max) 19 | a = append(a, s[:max/2-1]...) 20 | a = append(a, ".."...) 21 | a = append(a, s[len(s)-max/2+1:]...) 22 | return string(a) 23 | } 24 | 25 | // leftabbrev abbreviates string s by retaining its rightmost 'max' 26 | // characters. 27 | func leftabbrev(s string, max int) string { 28 | if len(s) <= max { 29 | return s 30 | } 31 | a := make([]byte, 0, max) 32 | a = append(a, ".."...) 33 | a = append(a, s[len(s)-max+2:]...) 34 | return string(a) 35 | } 36 | 37 | // trim returns a version of s where whitespace characters, as 38 | // defined by unicode.IsSpace, are replaced by ".". Space (" ") is 39 | // treated specially; it is allowed to remain. Adjacent whitespaces 40 | // are collapsed; leading and trailing whitespace is dropped 41 | // alltogether. 42 | func trimspace(s string) string { 43 | s = strings.TrimSpace(s) 44 | buf := make([]byte, 0, len(s)) 45 | var wasspace bool 46 | for width := 0; len(s) > 0; s = s[width:] { 47 | r := rune(s[0]) 48 | width = 1 49 | if r >= utf8.RuneSelf { 50 | r, width = utf8.DecodeRuneInString(s) 51 | } 52 | space := unicode.IsSpace(r) 53 | if space { 54 | if !wasspace { 55 | switch r { 56 | case ' ': 57 | buf = append(buf, " "...) 58 | default: 59 | buf = append(buf, "."...) 60 | } 61 | } 62 | } else { 63 | buf = append(buf, s[:width]...) 64 | } 65 | wasspace = space 66 | } 67 | return string(buf) 68 | } 69 | 70 | // trimpath trims a leading absolute path name, replacing all but its 71 | // base with ".." 72 | func trimpath(s string) string { 73 | u := s 74 | for len(s) > 0 && s[0] == '/' { 75 | i := strings.Index(s[1:], "/") 76 | if i < 0 || strings.IndexFunc(s[1:i], unicode.IsSpace) >= 0 { 77 | if s == "/" { 78 | s = u 79 | } 80 | return ".." + s 81 | } 82 | s, u = s[i+1:], s 83 | } 84 | return s 85 | } 86 | -------------------------------------------------------------------------------- /flow/strutil_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | import "testing" 8 | 9 | func TestAbbrev(t *testing.T) { 10 | if got, want := abbrev("dd if=/dev/zero of=$tmp/foobar count=1024.sleep 200&.sleep 200&.sleep 200", 10), "dd i.. 200"; got != want { 11 | t.Errorf("got %q, want %q", got, want) 12 | } 13 | } 14 | 15 | func TestLeftabbrev(t *testing.T) { 16 | for _, gw := range []struct{ s, want string }{ 17 | {"helloworld", "..rld"}, 18 | {"hello", "hello"}, 19 | {"ok", "ok"}, 20 | } { 21 | if got, want := leftabbrev(gw.s, 5), gw.want; got != want { 22 | t.Errorf("got %q, want %q", got, want) 23 | } 24 | } 25 | 26 | } 27 | 28 | func TestTrimspace(t *testing.T) { 29 | if got, want := trimspace("\t\t\nspace bound \n\t string\nok\n\n"), "space bound string.ok"; got != want { 30 | t.Errorf("got %q, want %q", got, want) 31 | } 32 | } 33 | 34 | func TestTrimpath(t *testing.T) { 35 | for _, gw := range []struct{ s, want string }{ 36 | {"/usr/bin/bwa -foo bar", "../bwa -foo bar"}, 37 | {"/usr/bin/echo", "../echo"}, 38 | {"/some/path/name/", "../name/"}, 39 | } { 40 | if got, want := trimpath(gw.s), gw.want; got != want { 41 | t.Errorf("got %q, want %q", got, want) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /flow/workingset.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package flow 6 | 7 | // Workingset tracks a set of pending flows and their 8 | // initial states. 9 | type workingset struct { 10 | pending map[*Flow]State 11 | counts [Max]int 12 | } 13 | 14 | func newWorkingset() *workingset { 15 | return &workingset{ 16 | pending: make(map[*Flow]State), 17 | } 18 | } 19 | 20 | // Add tracks the provided flow in the working set. Its state 21 | // is recorded. Add panics if a flow is added more than once. 22 | func (w *workingset) Add(f *Flow) { 23 | if w.Pending(f) { 24 | panic("flow already pending") 25 | } 26 | w.pending[f] = f.State 27 | w.counts[f.State]++ 28 | } 29 | 30 | // Pending returns whether the provided flow is tracked by 31 | // this working set. 32 | func (w *workingset) Pending(f *Flow) bool { 33 | _, ok := w.pending[f] 34 | return ok 35 | } 36 | 37 | // Done removes the provided flow from the working set. 38 | func (w *workingset) Done(f *Flow) { 39 | state, ok := w.pending[f] 40 | if !ok { 41 | panic("remove nonexistent flow") 42 | } 43 | delete(w.pending, f) 44 | w.counts[state]-- 45 | } 46 | 47 | // N returns the total number of pending flows tracked by 48 | // this working set. 49 | func (w *workingset) N() int { 50 | return len(w.pending) 51 | } 52 | 53 | // NState returns the number of pending flows in the provided 54 | // state. 55 | func (w *workingset) NState(state State) int { 56 | return w.counts[state] 57 | } 58 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/grailbio/reflow 2 | 3 | go 1.16 4 | 5 | require ( 6 | docker.io/go-docker v1.0.0 7 | github.com/Microsoft/go-winio v0.4.5 // indirect 8 | github.com/aws/aws-sdk-go v1.34.31 9 | github.com/aws/aws-xray-sdk-go v1.0.0-rc.2 10 | github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect 11 | github.com/docker/distribution v2.7.0+incompatible 12 | github.com/docker/docker v20.10.5+incompatible 13 | github.com/docker/go-connections v0.4.0 // indirect 14 | github.com/docker/go-units v0.4.0 15 | github.com/gogo/protobuf v1.3.2 // indirect 16 | github.com/google/go-containerregistry v0.0.0-20190116191554-efb7e1b888e1 17 | github.com/gorilla/context v1.1.1 // indirect 18 | github.com/gorilla/mux v1.6.2 // indirect 19 | github.com/grailbio/base v0.0.10 20 | github.com/grailbio/infra v0.0.2-0.20201105215413-8610bbcf04f2 21 | github.com/grailbio/testutil v0.0.4-0.20210412203346-39b3ca80f18a 22 | github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect 23 | github.com/morikuni/aec v1.0.0 // indirect 24 | github.com/opencontainers/go-digest v1.0.0 // indirect 25 | github.com/opencontainers/image-spec v1.0.1 // indirect 26 | github.com/sirupsen/logrus v1.3.0 // indirect 27 | github.com/spaolacci/murmur3 v1.1.0 // indirect 28 | github.com/willf/bloom v2.0.3+incompatible 29 | golang.org/x/net v0.0.0-20201021035429-f5854403a974 30 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 31 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 32 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a 33 | gonum.org/v1/gonum v0.0.0-20190902003836-43865b531bee 34 | gopkg.in/yaml.v2 v2.2.8 35 | gotest.tools v2.2.0+incompatible // indirect 36 | v.io/x/lib v0.1.5 37 | ) 38 | -------------------------------------------------------------------------------- /infra/infra_test.go: -------------------------------------------------------------------------------- 1 | package infra 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | "time" 7 | 8 | "github.com/grailbio/infra" 9 | ) 10 | 11 | func TestReflowletConfig(t *testing.T) { 12 | var schema = infra.Schema{ 13 | "reflowlet": new(ReflowletConfig), 14 | } 15 | for _, tt := range []struct { 16 | keyVal string 17 | want ReflowletConfig 18 | }{ 19 | { 20 | "reflowletconfig", 21 | DefaultReflowletConfig, 22 | }, 23 | { 24 | "reflowletconfig,maxidleduration=1m", 25 | MergeReflowletConfig(DefaultReflowletConfig, ReflowletConfig{ 26 | MaxIdleDuration: time.Minute}), 27 | }, 28 | { 29 | "reflowletconfig,lowthresholdpct=25.0", 30 | MergeReflowletConfig(DefaultReflowletConfig, ReflowletConfig{ 31 | VolumeWatcher: VolumeWatcher{LowThresholdPct: 25.0}}), 32 | }, 33 | { 34 | "reflowletconfig,lowthresholdpct=25.0,resizesleepduration=2s", 35 | MergeReflowletConfig(DefaultReflowletConfig, ReflowletConfig{ 36 | VolumeWatcher: VolumeWatcher{LowThresholdPct: 25.0, ResizeSleepDuration: 2 * time.Second}}), 37 | }, 38 | }{ 39 | config, err := schema.Make(infra.Keys{"reflowlet": tt.keyVal}) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | var rc *ReflowletConfig 44 | if err = config.Instance(&rc); err != nil { 45 | t.Fatal(err) 46 | } 47 | if got, want := *rc, tt.want; !reflect.DeepEqual(got, want) { 48 | t.Errorf("got %v, want %v", got, want) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /internal/ecrauth/ecrauth.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package ecrauth provides an interface and utilities for 6 | // authenticating AWS EC2 ECR Docker repositories. 7 | package ecrauth 8 | 9 | import ( 10 | "context" 11 | "fmt" 12 | 13 | "docker.io/go-docker/api/types" 14 | ) 15 | 16 | // Interface is the interface that is implemented by ECR 17 | // authentication providers. 18 | type Interface interface { 19 | // Authenticates tells whether this authenticator can authenticate the 20 | // provided image URI. 21 | Authenticates(ctx context.Context, image string) (bool, error) 22 | 23 | // Authenticate writes authentication information into the provided config struct. 24 | Authenticate(ctx context.Context, cfg *types.AuthConfig) error 25 | } 26 | 27 | // Login authenticates via the provided authenticator and then 28 | // returns the corresponding Docker login command. 29 | func Login(ctx context.Context, auth Interface) (string, error) { 30 | var cfg types.AuthConfig 31 | if err := auth.Authenticate(context.TODO(), &cfg); err != nil { 32 | return "", err 33 | } 34 | return fmt.Sprintf( 35 | "docker login -u %s -p %s https://%s", 36 | cfg.Username, cfg.Password, cfg.ServerAddress), nil 37 | } 38 | -------------------------------------------------------------------------------- /internal/fs/fs_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package fs 6 | 7 | import "syscall" 8 | 9 | // Usage specifies disk usage information. 10 | type Usage struct { 11 | // Total is the total number of bytes available on the disk. 12 | Total uint64 13 | 14 | // Free is the total number of bytes that are free on the disk. 15 | Free uint64 16 | 17 | // Avail is the total number of free bytes that are available to 18 | // unpriveliged users on the disk. 19 | Avail uint64 20 | } 21 | 22 | // Stat queries and returns disk usage information for the disk 23 | // at the given path. 24 | func Stat(path string) (Usage, error) { 25 | var stat syscall.Statfs_t 26 | if err := syscall.Statfs(path, &stat); err != nil { 27 | return Usage{}, err 28 | } 29 | 30 | size := uint64(stat.Bsize) 31 | return Usage{ 32 | Total: uint64(stat.Blocks) * size, 33 | Free: uint64(stat.Bfree) * size, 34 | Avail: uint64(stat.Bavail) * size, 35 | }, nil 36 | } 37 | -------------------------------------------------------------------------------- /internal/iputil/external.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package iputil 6 | 7 | import ( 8 | "errors" 9 | "net" 10 | ) 11 | 12 | // ExternalIP4 retrieves the machine's external (not necessarily public) 13 | // IP address. 14 | // 15 | // TODO(marius): use http://169.254.169.254/latest/meta-data/public-ipv4 on 16 | // AWS to get the public address as well. 17 | func ExternalIP4() (net.IP, error) { 18 | ifaces, err := net.Interfaces() 19 | if err != nil { 20 | return nil, err 21 | } 22 | for _, iface := range ifaces { 23 | if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 { 24 | continue 25 | } 26 | addrs, err := iface.Addrs() 27 | if err != nil { 28 | return nil, err 29 | } 30 | for _, addr := range addrs { 31 | var ip net.IP 32 | switch v := addr.(type) { 33 | case *net.IPNet: 34 | ip = v.IP 35 | case *net.IPAddr: 36 | ip = v.IP 37 | } 38 | if ip == nil || ip.IsLoopback() { 39 | continue 40 | } 41 | ip = ip.To4() 42 | if ip == nil { 43 | continue 44 | } 45 | return ip, nil 46 | } 47 | } 48 | return nil, errors.New("no external IPs found") 49 | } 50 | -------------------------------------------------------------------------------- /internal/scanner/README: -------------------------------------------------------------------------------- 1 | This is text/scanner from Go's standard library, 2 | with additional tokens for Reflow. 3 | -------------------------------------------------------------------------------- /internal/scanner/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package scanner_test 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | "text/scanner" 11 | ) 12 | 13 | func Example() { 14 | const src = ` 15 | // This is scanned code. 16 | if a > 10 { 17 | someParsable = text 18 | }` 19 | var s scanner.Scanner 20 | s.Filename = "example" 21 | s.Init(strings.NewReader(src)) 22 | var tok rune 23 | for tok != scanner.EOF { 24 | tok = s.Scan() 25 | fmt.Println("At position", s.Pos(), ":", s.TokenText()) 26 | } 27 | 28 | // Output: 29 | // At position example:3:4 : if 30 | // At position example:3:6 : a 31 | // At position example:3:8 : > 32 | // At position example:3:11 : 10 33 | // At position example:3:13 : { 34 | // At position example:4:15 : someParsable 35 | // At position example:4:17 : = 36 | // At position example:4:22 : text 37 | // At position example:5:3 : } 38 | // At position example:5:3 : 39 | } 40 | -------------------------------------------------------------------------------- /internal/walker/walker_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package walker 6 | 7 | import ( 8 | "os" 9 | "path/filepath" 10 | "reflect" 11 | "sort" 12 | "testing" 13 | 14 | "github.com/grailbio/testutil" 15 | ) 16 | 17 | func TestWalker(t *testing.T) { 18 | dir, cleanup := testutil.TempDir(t, "", "") 19 | defer cleanup() 20 | testutil.CreateDirectoryTree(t, dir, 2, 2, 10) 21 | dirlist, filelist := testutil.ListRecursively(t, dir) 22 | dirs := map[string]bool{} 23 | for _, dir := range dirlist { 24 | dirs[dir] = true 25 | } 26 | files := map[string]bool{} 27 | for _, file := range filelist { 28 | files[file] = true 29 | } 30 | var w Walker 31 | w.Init(dir) 32 | for w.Scan() { 33 | if w.Info().IsDir() { 34 | if !dirs[w.Path()] { 35 | t.Fatalf("unexpected directory %q", w.Path()) 36 | } 37 | delete(dirs, w.Path()) 38 | } else { 39 | if !files[w.Path()] { 40 | t.Fatalf("unexpected file %q", w.Path()) 41 | } 42 | delete(files, w.Path()) 43 | } 44 | } 45 | if err := w.Err(); err != nil { 46 | t.Fatal(err) 47 | } 48 | if got, want := dirs, map[string]bool{}; !reflect.DeepEqual(got, want) { 49 | t.Fatalf("got %v, want %v", got, want) 50 | } 51 | if got, want := files, map[string]bool{}; !reflect.DeepEqual(got, want) { 52 | t.Fatalf("got %v, want %v", got, want) 53 | } 54 | } 55 | 56 | func TestWalkerSymlinks(t *testing.T) { 57 | dir, cleanup := testutil.TempDir(t, "", "") 58 | defer cleanup() 59 | p := func(p ...string) string { 60 | p = append([]string{dir}, p...) 61 | return filepath.Join(p...) 62 | } 63 | if err := os.MkdirAll(p("dir"), 0777); err != nil { 64 | t.Fatal(err) 65 | } 66 | f, err := os.Create(p("dir", "file")) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | defer f.Close() 71 | if err := os.Symlink(p("dir"), p("link")); err != nil { 72 | t.Fatal(err) 73 | } 74 | var w Walker 75 | w.Init(dir) 76 | var paths []string 77 | for w.Scan() { 78 | paths = append(paths, w.Relpath()) 79 | } 80 | sort.Strings(paths) 81 | if got, want := paths, []string{".", "dir", "dir/file", "link", "link/file"}; !reflect.DeepEqual(got, want) { 82 | t.Errorf("got %v, want %v", got, want) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /lang/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package lang 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "text/scanner" 11 | ) 12 | 13 | // Error implements error reporting during parsing, typechecking, and 14 | // evaluation. It is not safe for concurrent access. 15 | type Error struct { 16 | W io.Writer // The io.Writer to which errors are reported. 17 | N int // The error count. 18 | } 19 | 20 | // Errorf formats and reports an error. 21 | func (e *Error) Errorf(pos scanner.Position, format string, args ...interface{}) { 22 | fmt.Fprintf(e.W, "%s: %s\n", pos, fmt.Sprintf(format, args...)) 23 | e.N++ 24 | } 25 | -------------------------------------------------------------------------------- /lang/lex_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package lang 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | ) 11 | 12 | func TestLexer(t *testing.T) { 13 | tests := []struct { 14 | body string 15 | toks []int 16 | }{ 17 | {`"hello world" intern 18 | // nothing should appear here 19 | someident`, []int{_EXPR, _INTERN, ';', _IDENT}}, 20 | {`foo(x) = img { 21 | Some Command here 22 | intern 23 | # whatever here 24 | } 25 | `, []int{_IDENT, '(', _IDENT, ')', '=', _IDENT, _TEMPLATE, ';'}}, 26 | {`let i = image("blah") in i { 27 | blah 28 | }`, []int{_LET, _IDENT, '=', _IMAGE, '(', _EXPR, ')', _IN, _IDENT, _TEMPLATE, ';'}}, 29 | } 30 | for i, test := range tests { 31 | lx := &Lexer{Body: bytes.NewReader([]byte(test.body)), Mode: LexerTop} 32 | lx.Init() 33 | var yy yySymType 34 | if got, want := lx.Lex(&yy), _STARTTOP; got != want { 35 | t.Fatalf("case %d: got %v, want %v", i, got, want) 36 | } 37 | for pos, tok := range test.toks { 38 | if got, want := lx.Lex(&yy), tok; got != want { 39 | t.Fatalf("case %d: got %v, want %v at position %d", i, got, want, pos) 40 | } 41 | } 42 | if got, want := lx.Lex(&yy), _EOF; got != want { 43 | t.Fatalf("case %d: expected EOF, got %v", i, got) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /lang/op_string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Code generated by "stringer -type=Op"; DO NOT EDIT 6 | 7 | package lang 8 | 9 | import "fmt" 10 | 11 | const _Op_name = "opIdentopApplyopConstopInternopExternopDefopImageopCollectopGroupbyopMapopFuncopVarfuncopExecopParamopConcatopPullupopLet" 12 | 13 | var _Op_index = [...]uint8{0, 7, 14, 21, 29, 37, 42, 49, 58, 67, 72, 78, 87, 93, 100, 108, 116, 121} 14 | 15 | func (i Op) String() string { 16 | i -= 1 17 | if i < 0 || i >= Op(len(_Op_index)-1) { 18 | return fmt.Sprintf("Op(%d)", i+1) 19 | } 20 | return _Op_name[_Op_index[i]:_Op_index[i+1]] 21 | } 22 | -------------------------------------------------------------------------------- /lang/template.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package lang 6 | 7 | import ( 8 | "errors" 9 | "strings" 10 | ) 11 | 12 | var errInvalidTemplate = errors.New("invalid template") 13 | 14 | // template implements a templating language: it parses identifiers 15 | // enclosed by '{{' and '}}'; these identifiers are placed (in order) 16 | // in Idents. 17 | type template struct { 18 | Idents []string 19 | Fragments []string 20 | } 21 | 22 | // newTemplate creates a new template from string s, 23 | // returning errInvalidTemplate on error. 24 | func newTemplate(s string) (*template, error) { 25 | s = strings.Replace(s, "%", "%%", -1) 26 | t := &template{} 27 | for { 28 | beg := strings.Index(s, `{{`) 29 | if beg < 0 { 30 | break 31 | } 32 | t.Fragments = append(t.Fragments, s[:beg]) 33 | if len(s) < 4 { 34 | return nil, errInvalidTemplate 35 | } 36 | s = s[beg+2:] 37 | end := strings.Index(s, `}}`) 38 | if end < 0 { 39 | return nil, errInvalidTemplate 40 | } 41 | t.Idents = append(t.Idents, s[:end]) 42 | s = s[end+2:] 43 | } 44 | t.Fragments = append(t.Fragments, s) 45 | return t, nil 46 | } 47 | 48 | func (t *template) Format(args ...string) string { 49 | s := "" 50 | for i := range t.Fragments { 51 | s += t.Fragments[i] 52 | if i >= len(t.Idents) { 53 | continue 54 | } 55 | s += args[i] 56 | } 57 | return s 58 | } 59 | -------------------------------------------------------------------------------- /lang/template_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package lang 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | ) 11 | 12 | func TestTemplate(t *testing.T) { 13 | tmpl, err := newTemplate(`hello%there {{foobar}}, {{ok}}`) 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | if got, want := tmpl.Idents, []string{"foobar", "ok"}; !reflect.DeepEqual(got, want) { 18 | t.Fatalf("got %v, want %v", got, want) 19 | } 20 | s := tmpl.Format("blah", "bloop") 21 | if got, want := s, "hello%%there blah, bloop"; got != want { 22 | t.Fatalf("got %q, want %q", got, want) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /lang/testdata/escape.reflow: -------------------------------------------------------------------------------- 1 | y = intern("s3://input") 2 | 3 | x = image("x") { 4 | echo % 5 | {{y}} 6 | } 7 | 8 | extern(x, "s3://output") 9 | 10 | -------------------------------------------------------------------------------- /lang/testdata/hash.reflow: -------------------------------------------------------------------------------- 1 | hash("v2") 2 | 3 | include("sub.reflow") 4 | extern(func(intern("file://blah")), "s3://blah") 5 | -------------------------------------------------------------------------------- /lang/testdata/intern.reflow: -------------------------------------------------------------------------------- 1 | a = intern(args) 2 | b = intern("s3://input") 3 | extern(pullup(a, b), "s3://output") 4 | -------------------------------------------------------------------------------- /lang/testdata/internlist.reflow: -------------------------------------------------------------------------------- 1 | a = intern("s3://bucket1/dir1/,s3://bucket2/dir2/,s3f://bucket3/file3") 2 | extern(a, "s3://output") 3 | -------------------------------------------------------------------------------- /lang/testdata/internlist2.reflow: -------------------------------------------------------------------------------- 1 | a = intern("s3f://bucket3/file,") 2 | extern(a, "s3://output") 3 | -------------------------------------------------------------------------------- /lang/testdata/internlistrewrite.reflow: -------------------------------------------------------------------------------- 1 | a = intern("xyz=s3://bucket1/dir1/,s3://bucket2/dir2/,thefile=s3f://bucket3/file3") 2 | extern(a, "s3://output") 3 | -------------------------------------------------------------------------------- /lang/testdata/internrewrite.reflow: -------------------------------------------------------------------------------- 1 | a = intern(args) 2 | b = intern("theinput=s3://bucket/input") 3 | extern(pullup(a, b), "s3://output") 4 | -------------------------------------------------------------------------------- /lang/testdata/main.reflow: -------------------------------------------------------------------------------- 1 | include("sub.reflow") 2 | 3 | extern(func(intern("file://blah")), "s3://blah") 4 | -------------------------------------------------------------------------------- /lang/testdata/param.reflow: -------------------------------------------------------------------------------- 1 | input = param("input", "the input directory") 2 | output = param("output", "the output directory") 3 | extern(intern(input), output) 4 | -------------------------------------------------------------------------------- /lang/testdata/sub.reflow: -------------------------------------------------------------------------------- 1 | func(x) = image("ubuntu") { 2 | echo {{x}} >$out 3 | } 4 | -------------------------------------------------------------------------------- /lang/type_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package lang 6 | 7 | import "testing" 8 | 9 | func TestFuncType(t *testing.T) { 10 | for arity := 0; arity < 10; arity++ { 11 | for rt := typeVoid; rt <= typeImage; rt++ { 12 | t1 := typeFunc(arity, rt) 13 | n, t2 := funcType(t1) 14 | if got, want := n, arity; got != want { 15 | t.Fatalf("got %v, want %v", got, want) 16 | } 17 | if got, want := t2, rt; got != want { 18 | t.Fatalf("got %v, want %v", got, want) 19 | } 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /liveset/bloomlive/bloomlive.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bloomlive 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | 11 | "github.com/grailbio/base/digest" 12 | "github.com/willf/bloom" 13 | ) 14 | 15 | // T implements a liveset.Liveset using a concrete bloom filter. 16 | // The bloom filter stores each digest according to its bytewise 17 | // representation. 18 | type T struct { 19 | *bloom.BloomFilter 20 | buf bytes.Buffer 21 | } 22 | 23 | // New creates a new T from a bloom filter. 24 | func New(b *bloom.BloomFilter) *T { 25 | return &T{BloomFilter: b} 26 | } 27 | 28 | // Contains tells whether the digest d is definitely in the set. Contains 29 | // is not safe to call concurrently. 30 | func (b *T) Contains(d digest.Digest) bool { 31 | b.buf.Reset() 32 | if _, err := digest.WriteDigest(&b.buf, d); err != nil { 33 | panic("failed to write digest " + d.String() + ": " + err.Error()) 34 | } 35 | return b.BloomFilter.Test(b.buf.Bytes()) 36 | } 37 | 38 | // MarshalJSON serializes the liveset into JSON. 39 | func (b *T) MarshalJSON() ([]byte, error) { 40 | return json.Marshal(b.BloomFilter) 41 | } 42 | 43 | // UnmarshalJSON deserializes the liveset from JSON. 44 | func (b *T) UnmarshalJSON(p []byte) error { 45 | b.BloomFilter = new(bloom.BloomFilter) 46 | return json.Unmarshal(p, b.BloomFilter) 47 | } 48 | -------------------------------------------------------------------------------- /liveset/liveset.go: -------------------------------------------------------------------------------- 1 | package liveset 2 | 3 | import ( 4 | "github.com/grailbio/base/digest" 5 | ) 6 | 7 | // A Liveset contains a possibly approximate judgement about live 8 | // objects. 9 | type Liveset interface { 10 | // Contains returns true if the given object definitely is in the 11 | // set; it may rarely return true when the object does not. 12 | Contains(digest.Digest) bool 13 | } 14 | -------------------------------------------------------------------------------- /local/cpufeatures_fallback.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !linux 6 | 7 | // Copyright 2018 GRAIL, Inc. All rights reserved. 8 | // Use of this source code is governed by the Apache 2.0 9 | // license that can be found in the LICENSE file. 10 | 11 | package local 12 | 13 | func cpuFeatures() ([]string, error) { 14 | return nil, nil 15 | } 16 | -------------------------------------------------------------------------------- /local/cpufeatures_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build linux 6 | 7 | // Copyright 2018 GRAIL, Inc. All rights reserved. 8 | // Use of this source code is governed by the Apache 2.0 9 | // license that can be found in the LICENSE file. 10 | 11 | package local 12 | 13 | import ( 14 | "bufio" 15 | "errors" 16 | "os" 17 | "strings" 18 | ) 19 | 20 | func cpuFeatures() ([]string, error) { 21 | f, err := os.Open("/proc/cpuinfo") 22 | if err != nil { 23 | return nil, err 24 | } 25 | defer f.Close() 26 | s := bufio.NewScanner(f) 27 | // Example features line: 28 | // flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke 29 | for s.Scan() && !strings.HasPrefix(s.Text(), "flags") { 30 | } 31 | if err := s.Err(); err != nil { 32 | return nil, err 33 | } 34 | parts := strings.SplitN(s.Text(), ":", 2) 35 | if len(parts) != 2 { 36 | return nil, errors.New("bad cpu flags") 37 | } 38 | flags := strings.Split(parts[1], " ") 39 | var features []string 40 | for _, flag := range flags { 41 | switch flag { 42 | case "avx": 43 | features = append(features, "intel_avx") 44 | case "avx2": 45 | features = append(features, "intel_avx2") 46 | case "avx512f": // AVX-512 foundation 47 | features = append(features, "intel_avx512") 48 | } 49 | } 50 | return features, nil 51 | } 52 | -------------------------------------------------------------------------------- /local/execstate_string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Code generated by "stringer -type=execState"; DO NOT EDIT 6 | 7 | package local 8 | 9 | import "fmt" 10 | 11 | const _execState_name = "execUnstartedexecInitexecCreatedexecRunningexecComplete" 12 | 13 | var _execState_index = [...]uint8{0, 13, 21, 32, 43, 55} 14 | 15 | func (i execState) String() string { 16 | if i < 0 || i >= execState(len(_execState_index)-1) { 17 | return fmt.Sprintf("execState(%d)", i) 18 | } 19 | return _execState_name[_execState_index[i]:_execState_index[i+1]] 20 | } 21 | -------------------------------------------------------------------------------- /local/manifest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package local 6 | 7 | import ( 8 | "time" 9 | 10 | "docker.io/go-docker/api/types" 11 | "github.com/grailbio/reflow" 12 | ) 13 | 14 | // execType defines the type of exec. It is used by the executor 15 | // to (re-) construct the correct type of exec from disk. 16 | type execType int 17 | 18 | const ( 19 | execDocker execType = iota 20 | execBlob 21 | ) 22 | 23 | // Manifest stores the state of an exec. It is serialized to JSON and 24 | // stored on disk so that executors are restartable, and can recover 25 | // from crashes. 26 | type Manifest struct { 27 | Type execType 28 | State execState 29 | PID int 30 | 31 | Created time.Time 32 | 33 | Result reflow.Result 34 | Config reflow.ExecConfig // The object config used to create this object. 35 | Docker types.ContainerJSON // Docker inspect output. 36 | Resources reflow.Resources 37 | Stats stats 38 | Gauges reflow.Gauges 39 | } 40 | -------------------------------------------------------------------------------- /local/oom_test.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | const dockerFmt = "2006-01-02T15:04:05.999999999Z" 11 | 12 | func TestOomTracker(t *testing.T) { 13 | testBootTime, err := time.Parse(dockerFmt, "2019-05-30T22:08:34.945074271Z") 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | start, err := time.Parse(dockerFmt, "2019-05-30T22:09:34.945074271Z") 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | // end is 11 minutes after start. 22 | end, err := time.Parse(dockerFmt, "2019-05-30T22:20:34.945074271Z") 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | f, err := ioutil.TempFile("", "oomlogs") 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | _, _ = f.WriteString(` 31 | 1,1,1,-;abc 32 | 1,1,30000000,-;Out of memory: Kill process 1 33 | 1,1,1,-;defg 34 | 1,1,1,-;hijk 35 | 1,1,180000000,-;Out of memory: Kill process 3 36 | 1,1,1,-;lmnop 37 | 1,1,1231234124312,-;Out of memory: Kill process 123456 38 | 1,1,180000000,-;Memory cgroup out of memory: Kill process 4`) 39 | if err := f.Close(); err != nil { 40 | t.Fatal(err) 41 | } 42 | oomTracker := newOOMTracker() 43 | ctx, cancel := context.WithCancel(context.Background()) 44 | defer cancel() 45 | oomTracker.monitor(ctx, nil, f.Name(), testBootTime) 46 | for _, tt := range []struct { 47 | pid int 48 | want bool 49 | }{ 50 | {1, false}, // PID OOMKilled before start. 51 | {2, false}, // PID never OOMKilled. 52 | {3, true}, // PID OOMKilled after start but before end. 53 | {4, true}, // PID cgroup memory limit exceeded after start but before end. 54 | } { 55 | ok, _ := oomTracker.Oom(tt.pid, start, end) 56 | if got, want := ok, tt.want; got != want { 57 | t.Fatalf("got %v, want %v", got, want) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /local/pool_test.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/grailbio/base/digest" 9 | "github.com/grailbio/reflow" 10 | "github.com/grailbio/reflow/pool" 11 | "github.com/grailbio/reflow/taskdb" 12 | ) 13 | 14 | type mockTaskDB struct { 15 | taskdb.TaskDB 16 | rid digest.Digest 17 | r reflow.Resources 18 | kaid digest.Digest 19 | kaN int 20 | endId digest.Digest 21 | endTimeN int 22 | } 23 | 24 | func (db *mockTaskDB) SetResources(ctx context.Context, id digest.Digest, resources reflow.Resources) error { 25 | db.rid = id 26 | db.r = resources 27 | return nil 28 | } 29 | 30 | func (db *mockTaskDB) KeepIDAlive(ctx context.Context, id digest.Digest, keepalive time.Time) error { 31 | db.kaid = id 32 | db.kaN += 1 33 | return nil 34 | } 35 | 36 | func (db *mockTaskDB) SetEndTime(ctx context.Context, id digest.Digest, end time.Time) (err error) { 37 | db.endId = id 38 | db.endTimeN += 1 39 | return 40 | } 41 | 42 | func TestMaintainTaskDBRow(t *testing.T) { 43 | tdb := &mockTaskDB{} 44 | r := reflow.Resources{"mem": 10, "cpu": 1, "disk": 20} 45 | id := reflow.NewStringDigest("testpool") 46 | p := Pool{ResourcePool: pool.NewResourcePool(nil, nil), TaskDBPoolId: id, TaskDB: tdb} 47 | p.Init(r, nil) 48 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 49 | done := make(chan struct{}) 50 | go func() { 51 | p.MaintainTaskDBRow(ctx) 52 | done <- struct{}{} 53 | }() 54 | time.Sleep(100 * time.Millisecond) 55 | cancel() 56 | <-done 57 | if got, want := tdb.r, r; !got.Equal(want) { 58 | t.Errorf("got %v, want %v", got, want) 59 | } 60 | if got, want := tdb.kaN, 1; got != want { 61 | t.Errorf("got %v, want %v", got, want) 62 | } 63 | if got, want := tdb.endTimeN, 1; got != want { 64 | t.Errorf("got %v, want %v", got, want) 65 | } 66 | if got, want := tdb.rid, id.Digest(); got != want { 67 | t.Errorf("got %v, want %v", got, want) 68 | } 69 | if got, want := tdb.kaid, id.Digest(); got != want { 70 | t.Errorf("got %v, want %v", got, want) 71 | } 72 | if got, want := tdb.endId, id.Digest(); got != want { 73 | t.Errorf("got %v, want %v", got, want) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /local/pooltest/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package pooltest tests pools. (And this file is here 6 | // only to make the build happy.) 7 | package pooltest 8 | -------------------------------------------------------------------------------- /local/pooltest/pool_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !unit integration 6 | 7 | package pooltest 8 | 9 | import ( 10 | "testing" 11 | 12 | "github.com/grailbio/reflow/local" 13 | . "github.com/grailbio/reflow/local/testutil" 14 | "github.com/grailbio/testutil" 15 | ) 16 | 17 | // TestAlloc tests the whole lifetime of an alloc. One is created, an 18 | // exec is created; it is then not maintained and is garbage 19 | // collected. It becomes a zombie. 20 | func TestAlloc(t *testing.T) { 21 | p, cleanup := NewTestPoolOrSkip(t) 22 | defer cleanup() 23 | TestPool(t, p) 24 | } 25 | 26 | func TestPoolFailsLessThanExpectedMem(t *testing.T) { 27 | // We put this in /tmp because it's one of the default locations 28 | // that are bindable from Docker for Mac. 29 | dir, cleanup := testutil.TempDir(t, "/tmp", "reflowtest") 30 | defer cleanup() 31 | p := &local.Pool{ 32 | Client: NewDockerClientOrSkip(t), 33 | Dir: dir, 34 | } 35 | err := p.Start(10 << 40 /* 10TiB, insane expectation */) 36 | if err != nil { 37 | t.Logf("expected error observed: %v", err) 38 | } else { 39 | t.Fatalf("pool did not fail") 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /local/stats_test.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestStats(t *testing.T) { 10 | const ( 11 | n = 10 12 | name = "some_stat" 13 | ) 14 | var ( 15 | st = make(stats) 16 | times = make([]time.Time, n) 17 | ) 18 | for i := 0; i < n; i++ { 19 | times[i] = time.Now().Add(time.Duration(i) * time.Second) 20 | st.Observe(times[i], name, float64(i)) 21 | } 22 | first, last := times[0], times[n-1] 23 | rand.Shuffle(len(times), func(i, j int) { 24 | times[i], times[j] = times[j], times[i] 25 | }) 26 | if got, want := st.N(name), int64(n); got != want { 27 | t.Fatalf("got %v, want %v", got, want) 28 | } 29 | if got, want := st.First(name), first; got != want { 30 | t.Fatalf("got %v, want %v", got, want) 31 | } 32 | if got, want := st.Last(name), last; got != want { 33 | t.Fatalf("got %v, want %v", got, want) 34 | } 35 | if got, want := st.Max(name), float64(n-1); got != want { 36 | t.Fatalf("got %v, want %v", got, want) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /metrics/README.md: -------------------------------------------------------------------------------- 1 | # Reflow Metrics Library 2 | 3 | The Reflow metrics library should be used to provide observability into 4 | Reflow. 5 | 6 | ## Defining metrics 7 | 8 | Metrics are defined in `metrics.yaml`. Each named dictionary in the 9 | `metrics.yaml` root array will produce a metric. Note that the name for 10 | a metric must be given in snake_case and contain only lowercase alpha 11 | characters. Metric configuration dictionaries can have the following 12 | fields: 13 | 14 | - Type 15 | - The following types of metrics are available. 16 | - `counter` - counters can only be incremented or reset to zero. 17 | - `gauge` - gauges can be incremented, decremented or set to an 18 | arbitrary value. 19 | - `histogram` - histograms can used to observe arbitrary values 20 | (discretized to a set of buckets configured at compilation). 21 | 22 | - Help 23 | - All metrics should be documented with a helpful help message. This 24 | field can be used by clients to provide documentation at query-time. 25 | 26 | - Labels 27 | - Labels give dimensionality to metrics. Labels are written in 28 | snake_case and can only contain lowercase alpha characters. 29 | 30 | - Buckets 31 | - Buckets is a special field available on the "histogram" metric type. 32 | Observations will be discretized to fit into these buckets, based on 33 | the implementation of the metrics client. 34 | 35 | ## Regenerating metrics.go 36 | 37 | After updating metrics.json, use the genmetrics utility binary to 38 | generate static the go package functions that are called to provide 39 | observations. 40 | 41 | `go run cmd/genmetrics/main.go metrics/metrics.yaml metrics` 42 | -------------------------------------------------------------------------------- /metrics/nopclient.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package metrics 6 | 7 | import "github.com/grailbio/infra" 8 | 9 | type nopGauge struct{} 10 | 11 | func (n *nopGauge) Set(float64) {} 12 | 13 | func (n *nopGauge) Inc() {} 14 | 15 | func (n *nopGauge) Dec() {} 16 | 17 | func (n *nopGauge) Add(float64) {} 18 | 19 | func (n *nopGauge) Sub(float64) {} 20 | 21 | type nopCounter struct{} 22 | 23 | func (n *nopCounter) Inc() {} 24 | 25 | func (n *nopCounter) Add(float64) {} 26 | 27 | type nopHistogram struct{} 28 | 29 | func (n *nopHistogram) Observe(float64) {} 30 | 31 | type nopClient struct{} 32 | 33 | // GetGauge returns a nopGauge which does nothing. 34 | func (*nopClient) GetGauge(string, map[string]string) Gauge { 35 | return &nopGauge{} 36 | } 37 | 38 | // GetCounter returns a nopCounter which does nothing. 39 | func (*nopClient) GetCounter(string, map[string]string) Counter { 40 | return &nopCounter{} 41 | } 42 | 43 | // GetHistogram returns a nopHistogram which does nothing. 44 | func (*nopClient) GetHistogram(string, map[string]string) Histogram { 45 | return &nopHistogram{} 46 | } 47 | 48 | func init() { 49 | infra.Register("nopmetrics", new(nopClient)) 50 | } 51 | 52 | // Init implements infra.Provider and gets called when an instance of nopClient 53 | // is created with infra.Config.Instance(...) 54 | func (nopClient) Init() error { 55 | return nil 56 | } 57 | 58 | // Help implements infra.Provider 59 | func (nopClient) Help() string { 60 | return "nopClient does nothing, use it when you don't want any metrics" 61 | } 62 | -------------------------------------------------------------------------------- /pool/resourcepool_test.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/grailbio/reflow/log" 9 | ) 10 | 11 | func newInspectAlloc(p *ResourcePool, id string, ka time.Duration) *inspectAlloc { 12 | a := &inspectAlloc{ 13 | Alloc: idAlloc(id), 14 | inspect: AllocInspect{ 15 | Created: time.Now(), 16 | Expires: time.Now().Add(ka), 17 | }, 18 | } 19 | p.allocs[id] = a 20 | return a 21 | } 22 | 23 | func TestStopIfIdleFor(t *testing.T) { 24 | p := ResourcePool{log: log.Std} 25 | if stopped, _ := p.StopIfIdleFor(time.Second); !stopped { 26 | t.Fatal("idle pool must be stopped") 27 | } 28 | p.allocs = map[string]Alloc{} 29 | a1 := newInspectAlloc(&p, "alloc1", 100*time.Millisecond) 30 | stopped, _ := p.StopIfIdleFor(time.Second) 31 | if stopped { 32 | t.Fatal("busy pool must not be stopped") 33 | } 34 | _, _ = a1.Keepalive(context.Background(), time.Minute) 35 | stopped, tte := p.StopIfIdleFor(time.Second) 36 | if stopped { 37 | t.Fatal("busy pool must not be stopped") 38 | } 39 | if tte > 60*time.Second || tte < 59*time.Second { 40 | t.Fatalf("got %s, want ~1min", tte) 41 | } 42 | a2 := newInspectAlloc(&p, "alloc2", 100*time.Millisecond) 43 | _, _ = a2.Keepalive(context.Background(), 5*time.Minute) 44 | stopped, tte = p.StopIfIdleFor(time.Second) 45 | if stopped { 46 | t.Fatal("busy pool must not be stopped") 47 | } 48 | if tte > 300*time.Second || tte < 299*time.Second { 49 | t.Fatalf("got %s, want ~5min", tte) 50 | } 51 | p.allocs = map[string]Alloc{} // clear all allocs 52 | if stopped, _ := p.StopIfIdleFor(time.Second); !stopped { 53 | t.Fatal("idle pool must be stopped") 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /predictor/taskgroup_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package predictor 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/grailbio/reflow" 11 | "github.com/grailbio/reflow/sched" 12 | "github.com/grailbio/reflow/taskdb" 13 | ) 14 | 15 | func TestGetTaskGroups(t *testing.T) { 16 | var ( 17 | img = "image" 18 | cmd = "cmd" 19 | ident = "ident" 20 | imgCmdID = taskdb.NewImgCmdID(img, cmd) 21 | task = &sched.Task{ 22 | Config: reflow.ExecConfig{ 23 | Image: img, 24 | Cmd: cmd, 25 | Ident: ident, 26 | }, 27 | } 28 | ) 29 | 30 | taskGroups := getTaskGroups(task) 31 | if got, want := len(taskGroups), 2; got != want { 32 | t.Fatalf("got %d taskgroups, want %d", got, want) 33 | } 34 | 35 | imgcmdgroup := taskGroups[0] 36 | if _, ok := imgcmdgroup.(imgCmdGroup); !ok { 37 | t.Fatalf("imgcmd group type: got %T, want %T", imgcmdgroup, imgCmdGroup{}) 38 | } 39 | if got, want := imgcmdgroup.Name(), "imgCmdGroup:"+imgCmdID.ID(); got != want { 40 | t.Errorf("imgcmd group name: got %s, want %s", got, want) 41 | } 42 | imgcmdQuery := imgcmdgroup.Query() 43 | if got, want := imgcmdQuery.ImgCmdID.ID(), imgCmdID.ID(); got != want { 44 | t.Errorf("imgcmd group query imgcmdid: got %s, want %s", got, want) 45 | } 46 | if got, want := imgcmdQuery.Limit, queryLimit; got != want { 47 | t.Errorf("imgcmd group query limit: got %d, want %d", got, want) 48 | } 49 | 50 | identgroup := taskGroups[1] 51 | if _, ok := identgroup.(identGroup); !ok { 52 | t.Fatalf("ident group type: got %T, want %T", identgroup, identGroup{}) 53 | } 54 | if got, want := identgroup.Name(), "identGroup:"+ident; got != want { 55 | t.Errorf("ident group name: got %s, want %s", got, want) 56 | } 57 | identQuery := identgroup.Query() 58 | if got, want := identQuery.Ident, ident; got != want { 59 | t.Errorf("ident group query ident: got %s, want %s", got, want) 60 | } 61 | if got, want := identQuery.Limit, queryLimit; got != want { 62 | t.Errorf("identQuery group query limit: got %d, want %d", got, want) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /repository/blobrepo/dial.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package blobrepo 6 | 7 | import ( 8 | "context" 9 | "net/url" 10 | "sync" 11 | 12 | "github.com/grailbio/reflow" 13 | "github.com/grailbio/reflow/blob" 14 | "github.com/grailbio/reflow/repository" 15 | ) 16 | 17 | var ( 18 | mu sync.RWMutex 19 | mux = make(blob.Mux) 20 | ) 21 | 22 | // Register registers a blob store implementation used to dial 23 | // repositories for the provided scheme. 24 | func Register(scheme string, store blob.Store) { 25 | mu.Lock() 26 | mux[scheme] = store 27 | mu.Unlock() 28 | repository.RegisterScheme(scheme, Dial) 29 | } 30 | 31 | // Dial dials a blob repository. The URL must have the form: 32 | // 33 | // type://bucket/prefix 34 | // 35 | // TODO(marius): we should support shipping authentication 36 | // information in the URL also. 37 | func Dial(u *url.URL) (reflow.Repository, error) { 38 | mu.RLock() 39 | bucket, prefix, err := mux.Bucket(context.Background(), u.String()) 40 | mu.RUnlock() 41 | if err != nil { 42 | return nil, err 43 | } 44 | return &Repository{bucket, prefix}, nil 45 | } 46 | -------------------------------------------------------------------------------- /repository/filerepo/walker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package filerepo 6 | 7 | import ( 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/grailbio/base/digest" 12 | "github.com/grailbio/reflow" 13 | fswalker "github.com/grailbio/reflow/internal/walker" 14 | ) 15 | 16 | type walker struct { 17 | repo *Repository 18 | walker fswalker.Walker 19 | dgst digest.Digest 20 | err error 21 | } 22 | 23 | func (w *walker) Init(r *Repository) { 24 | w.walker.Init(r.Root) 25 | } 26 | 27 | func (w *walker) Digest() digest.Digest { 28 | return w.dgst 29 | } 30 | 31 | func (w *walker) Err() error { 32 | if w.err != nil { 33 | return w.err 34 | } 35 | return w.walker.Err() 36 | } 37 | 38 | func (w *walker) Path() string { 39 | return w.walker.Path() 40 | } 41 | 42 | func (w *walker) Info() os.FileInfo { 43 | return w.walker.Info() 44 | } 45 | 46 | func (w *walker) Scan() bool { 47 | for { 48 | if !w.walker.Scan() { 49 | return false 50 | } 51 | if w.walker.Info().IsDir() { 52 | continue 53 | } 54 | path := w.Path() 55 | first, last := filepath.Base(filepath.Dir(path)), filepath.Base(path) 56 | if first == "tmp" { 57 | continue 58 | } 59 | w.dgst, w.err = reflow.Digester.Parse(first + last) 60 | if w.err != nil { 61 | return false 62 | } 63 | return true 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /repository/http/dial.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package http 6 | 7 | import ( 8 | "net/http" 9 | "net/url" 10 | "sync" 11 | 12 | "github.com/grailbio/reflow" 13 | "github.com/grailbio/reflow/errors" 14 | "github.com/grailbio/reflow/repository" 15 | "github.com/grailbio/reflow/repository/client" 16 | "github.com/grailbio/reflow/rest" 17 | ) 18 | 19 | func init() { 20 | repository.RegisterScheme("https", Dial) 21 | } 22 | 23 | // HTTPClient is the client that is used to instantiate the (REST) 24 | // API client for remote repositories. 25 | var HTTPClient *http.Client 26 | 27 | var ( 28 | mu sync.Mutex 29 | // clients caches repository clients. 30 | clients = map[string]*client.Client{} 31 | ) 32 | 33 | // Dial implements repository dialling for https urls. 34 | func Dial(u *url.URL) (reflow.Repository, error) { 35 | if u.Scheme != "https" { 36 | return nil, errors.E("dial", u.String(), errors.NotSupported, errors.Errorf("unknown scheme %q", u.Scheme)) 37 | } 38 | mu.Lock() 39 | defer mu.Unlock() 40 | key := u.String() 41 | if clients[key] == nil { 42 | clients[key] = &client.Client{Client: rest.NewClient(HTTPClient, u, nil)} 43 | } 44 | return clients[key], nil 45 | } 46 | -------------------------------------------------------------------------------- /repository/missing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package repository 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/grailbio/reflow" 11 | "github.com/grailbio/reflow/errors" 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | // Missing returns the files in files that are missing from 16 | // repository r. Missing returns an error if any underlying 17 | // call fails. 18 | func Missing(ctx context.Context, r reflow.Repository, files ...reflow.File) ([]reflow.File, error) { 19 | exists := make([]bool, len(files)) 20 | g, _ := errgroup.WithContext(ctx) 21 | for _, file := range files { 22 | if file.IsRef() { 23 | return nil, errors.E("missing", errors.Invalid, errors.Errorf("unresolved file: %v", file)) 24 | } 25 | } 26 | for i, file := range files { 27 | i, file := i, file 28 | g.Go(func() (err error) { 29 | if _, err = r.Stat(ctx, file.ID); err == nil { 30 | exists[i] = true 31 | } else if errors.Is(errors.NotExist, err) { 32 | err = nil 33 | } 34 | return err 35 | }) 36 | } 37 | if err := g.Wait(); err != nil { 38 | return nil, err 39 | } 40 | if err := ctx.Err(); err != nil { 41 | return nil, err 42 | } 43 | all := files 44 | files = nil 45 | for i := range exists { 46 | if !exists[i] { 47 | files = append(files, all[i]) 48 | } 49 | } 50 | return files, nil 51 | } 52 | -------------------------------------------------------------------------------- /repository/noprepo/noprepo.go: -------------------------------------------------------------------------------- 1 | package noprepo 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/url" 7 | "strings" 8 | "time" 9 | 10 | "github.com/grailbio/base/digest" 11 | "github.com/grailbio/infra" 12 | "github.com/grailbio/reflow" 13 | "github.com/grailbio/reflow/blob/nopblob" 14 | "github.com/grailbio/reflow/liveset" 15 | "github.com/grailbio/reflow/repository/blobrepo" 16 | ) 17 | 18 | func init() { 19 | infra.Register("noprepo", new(Repository)) 20 | } 21 | 22 | // Repository is a no-op Repository provider. 23 | type Repository struct { 24 | reflow.Repository 25 | } 26 | 27 | // Init implements infra.Provider. 28 | func (r *Repository) Init() error { 29 | r.Repository = InitRepo() 30 | return nil 31 | } 32 | 33 | func InitRepo() reflow.Repository { 34 | blob := nopblob.NewStore() 35 | blobrepo.Register("nop", blob) 36 | return &Repository{} 37 | } 38 | 39 | func (r *Repository) Collect(context.Context, liveset.Liveset) error { 40 | return nil 41 | } 42 | 43 | func (r *Repository) CollectWithThreshold(ctx context.Context, live liveset.Liveset, dead liveset.Liveset, threshold time.Time, dryrun bool) error { 44 | return nil 45 | } 46 | 47 | func (r *Repository) Stat(context.Context, digest.Digest) (reflow.File, error) { 48 | return reflow.File{}, nil 49 | } 50 | 51 | func (r *Repository) Get(context.Context, digest.Digest) (io.ReadCloser, error) { 52 | return io.NopCloser(strings.NewReader("")), nil 53 | } 54 | 55 | func (r *Repository) Put(context.Context, io.Reader) (digest.Digest, error) { 56 | return digest.Digest{}, nil 57 | } 58 | 59 | func (r *Repository) WriteTo(context.Context, digest.Digest, *url.URL) error { 60 | return nil 61 | } 62 | 63 | func (r *Repository) ReadFrom(context.Context, digest.Digest, *url.URL) error { 64 | return nil 65 | } 66 | 67 | func (r *Repository) URL() *url.URL { 68 | return &url.URL{Scheme: "nop"} 69 | } 70 | -------------------------------------------------------------------------------- /repository/s3/repo_test.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go/aws/session" 8 | "github.com/grailbio/infra" 9 | _ "github.com/grailbio/infra/aws" 10 | "github.com/grailbio/reflow" 11 | "github.com/grailbio/reflow/log" 12 | "github.com/grailbio/reflow/test/testutil" 13 | ) 14 | 15 | func TestS3RepositoryInfra(t *testing.T) { 16 | const bucket = "reflow-unittest" 17 | testutil.SkipIfNoCreds(t) 18 | var schema = infra.Schema{ 19 | "session": new(session.Session), 20 | "logger": new(log.Logger), 21 | "repository": new(reflow.Repository), 22 | } 23 | config, err := schema.Make(infra.Keys{ 24 | "session": "awssession", 25 | "logger": "logger", 26 | "repository": fmt.Sprintf("s3,bucket=%v", bucket), 27 | }) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | 32 | var repo reflow.Repository 33 | config.Must(&repo) 34 | var s3repos *Repository 35 | config.Must(&s3repos) 36 | if got, want := s3repos.BucketName, bucket; got != want { 37 | t.Errorf("got %v, want %v", s3repos.BucketName, bucket) 38 | } 39 | if got, want := repo.URL().String(), "s3://reflow-unittest/"; got != want { 40 | t.Errorf("got %v, want %v", got, want) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /repository/s3/test/fakes3.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "github.com/grailbio/infra" 5 | "github.com/grailbio/reflow" 6 | ) 7 | 8 | func init() { 9 | infra.Register("fakes3", new(Repository)) 10 | } 11 | 12 | // Repository is a fake repos infra provider and should be used only in tests. 13 | type Repository struct { 14 | reflow.Repository 15 | } 16 | -------------------------------------------------------------------------------- /repository/test/calltype_string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Code generated by "stringer -type=callType"; DO NOT EDIT 6 | 7 | package test 8 | 9 | import "fmt" 10 | 11 | const _callType_name = "callStatreturnStatcallGetreturnGetcallPutreturnPutcallWriteToreturnWriteTocallReadFromreturnReadFrom" 12 | 13 | var _callType_index = [...]uint8{0, 8, 18, 25, 34, 41, 50, 61, 74, 86, 100} 14 | 15 | func (i callType) String() string { 16 | if i < 0 || i >= callType(len(_callType_index)-1) { 17 | return fmt.Sprintf("callType(%d)", i) 18 | } 19 | return _callType_name[_callType_index[i]:_callType_index[i+1]] 20 | } 21 | -------------------------------------------------------------------------------- /repository/test/inmemrepo.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/grailbio/infra" 8 | "github.com/grailbio/reflow" 9 | "github.com/grailbio/reflow/blob" 10 | "github.com/grailbio/reflow/blob/testblob" 11 | infra2 "github.com/grailbio/reflow/infra" 12 | "github.com/grailbio/reflow/repository/blobrepo" 13 | ) 14 | 15 | func init() { 16 | infra.Register("inmemrepo", new(InMemoryBlobRepository)) 17 | } 18 | 19 | const scheme = "inmemory" 20 | 21 | // InMemoryBlobRepository is a blob backed repository which stores values in memory 22 | type InMemoryBlobRepository struct { 23 | *blobrepo.Repository 24 | infra2.BucketNameFlagsTrait 25 | } 26 | 27 | var ( 28 | store blob.Store 29 | registerOnce sync.Once 30 | ) 31 | 32 | // Init implements infra.Provider 33 | func (r *InMemoryBlobRepository) Init() error { 34 | registerOnce.Do(func() { 35 | store = testblob.New(scheme) 36 | blobrepo.Register(scheme, store) 37 | }) 38 | bucket, err := store.Bucket(context.Background(), r.BucketNameFlagsTrait.BucketName) 39 | if err != nil { 40 | return err 41 | } 42 | r.Repository = &blobrepo.Repository{Bucket: bucket} 43 | return nil 44 | } 45 | 46 | func GetFiles(ctx context.Context, bucketName string) ([]reflow.File, error) { 47 | bucket, err := store.Bucket(ctx, bucketName) 48 | if err != nil { 49 | return nil, err 50 | } 51 | const withMetadata = true 52 | scan := bucket.Scan("", withMetadata) 53 | var files []reflow.File 54 | for scan.Scan(ctx) { 55 | _, file := scan.Key(), scan.File() 56 | files = append(files, file) 57 | } 58 | return files, nil 59 | } 60 | -------------------------------------------------------------------------------- /runner/cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package runner 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/grailbio/reflow" 11 | "github.com/grailbio/reflow/pool" 12 | ) 13 | 14 | // Cluster is a kind of pool.Pool that also allows the user to 15 | // directly reserve an alloc. This way, the cluster can be responsive 16 | // to demand. 17 | type Cluster interface { 18 | pool.Pool 19 | 20 | // Allocate reserves an alloc of at least min, and at most max resources. 21 | // The cluster may scale elastically in order to meet this demand. 22 | // Labels are passed down to the underlying pool. 23 | Allocate(ctx context.Context, req reflow.Requirements, labels pool.Labels) (pool.Alloc, error) 24 | 25 | // CanAllocate returns whether this cluster can allocate the given amount of resources. 26 | CanAllocate(reflow.Resources) (bool, error) 27 | 28 | // MaxAlloc returns the max resources which can be obtained in a single alloc from this cluster. 29 | MaxAlloc() reflow.Resources 30 | 31 | // GetName returns the name of the cluster 32 | GetName() string 33 | } 34 | 35 | // TracingCluster is a cluster that traces the actions of an underlying 36 | // cluster manager. 37 | type TracingCluster struct { 38 | Cluster 39 | } 40 | -------------------------------------------------------------------------------- /runner/phase_string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Code generated by "stringer -type=Phase"; DO NOT EDIT 6 | 7 | package runner 8 | 9 | import "fmt" 10 | 11 | const _Phase_name = "InitEvalRetryDone" 12 | 13 | var _Phase_index = [...]uint8{0, 4, 8, 13, 17} 14 | 15 | func (i Phase) String() string { 16 | if i < 0 || i >= Phase(len(_Phase_index)-1) { 17 | return fmt.Sprintf("Phase(%d)", i) 18 | } 19 | return _Phase_name[_Phase_index[i]:_Phase_index[i+1]] 20 | } 21 | -------------------------------------------------------------------------------- /runtime/resolver_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go/aws/session" 8 | "github.com/grailbio/infra" 9 | _ "github.com/grailbio/infra/aws" 10 | "github.com/grailbio/reflow/ec2authenticator" 11 | "github.com/grailbio/reflow/test/testutil" 12 | ) 13 | 14 | func TestResolveImages(t *testing.T) { 15 | if testing.Short() { 16 | t.Skip("requires network access") 17 | } 18 | testutil.SkipIfNoCreds(t) 19 | var schema = infra.Schema{ 20 | "session": new(session.Session), 21 | } 22 | config, err := schema.Make(infra.Keys{ 23 | "session": "awssession", 24 | }) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | var sess *session.Session 29 | config.Must(&sess) 30 | const ( 31 | repository = "619867110810.dkr.ecr.us-west-2.amazonaws.com" 32 | resolved = "619867110810.dkr.ecr.us-west-2.amazonaws.com/awstool@sha256:df811e08963f5e3ebba7efc8a003ec6fbde401ea272dd34378a9f2aa24b708db" 33 | ) 34 | r := ImageResolver{Authenticator: ec2authenticator.New(sess)} 35 | for _, img := range []string{ 36 | repository + "/awstool", 37 | repository + "/awstool$aws", 38 | repository + "/awstool$docker", 39 | repository + "/awstool$aws$docker", 40 | repository + "/awstool$docker$aws", 41 | } { 42 | canonical, err := r.ResolveImages(context.Background(), []string{img}) 43 | if err != nil { 44 | t.Errorf("error while getting canonical name for %s: %v", img, err) 45 | continue 46 | } 47 | if got, want := canonical[img], resolved; got != want { 48 | t.Errorf("expected %s, got %s", want, got) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /runtime/scheduler.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/grailbio/infra" 8 | "github.com/grailbio/reflow" 9 | "github.com/grailbio/reflow/errors" 10 | "github.com/grailbio/reflow/log" 11 | "github.com/grailbio/reflow/repository" 12 | "github.com/grailbio/reflow/sched" 13 | "github.com/grailbio/reflow/taskdb" 14 | ) 15 | 16 | const ( 17 | // The amount of outstanding number of transfers 18 | // between each repository. 19 | defaultTransferLimit = 20 20 | 21 | // The number of concurrent stat operations that can 22 | // be performed against a repository. 23 | statLimit = 200 24 | ) 25 | 26 | // newScheduler returns a new scheduler with the specified configuration. 27 | // Cancelling the returned context.CancelFunc stops the scheduler. 28 | func newScheduler(config infra.Config, logger *log.Logger) (*sched.Scheduler, error) { 29 | var ( 30 | err error 31 | tdb taskdb.TaskDB 32 | repo reflow.Repository 33 | limit int 34 | ) 35 | if err = config.Instance(&tdb); err != nil { 36 | if !strings.HasPrefix(err.Error(), "no providers for type taskdb.TaskDB") { 37 | return nil, err 38 | } 39 | logger.Debug(err) 40 | } 41 | if err = config.Instance(&repo); err != nil { 42 | return nil, err 43 | } 44 | if limit, err = transferLimit(config); err != nil { 45 | return nil, err 46 | } 47 | transferer := &repository.Manager{ 48 | Status: nil, 49 | PendingTransfers: repository.NewLimits(limit), 50 | Stat: repository.NewLimits(statLimit), 51 | Log: logger, 52 | } 53 | if repo != nil { 54 | transferer.PendingTransfers.Set(repo.URL().String(), int(^uint(0)>>1)) 55 | } 56 | scheduler := sched.New() 57 | 58 | scheduler.Transferer = transferer 59 | scheduler.Log = logger.Tee(nil, "scheduler: ") 60 | scheduler.TaskDB = tdb 61 | scheduler.ExportStats() 62 | 63 | return scheduler, nil 64 | } 65 | 66 | // TransferLimit returns the configured transfer limit. 67 | func transferLimit(config infra.Config) (int, error) { 68 | lim := config.Value("transferlimit") 69 | if lim == nil { 70 | return defaultTransferLimit, nil 71 | } 72 | v, ok := lim.(int) 73 | if !ok { 74 | return 0, errors.New(fmt.Sprintf("non-integer limit %v", lim)) 75 | } 76 | return v, nil 77 | } 78 | -------------------------------------------------------------------------------- /runtime/scheduler_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/grailbio/reflow/log" 9 | "github.com/grailbio/reflow/repository" 10 | "github.com/grailbio/reflow/runner" 11 | "github.com/grailbio/reflow/test/infra" 12 | ) 13 | 14 | func TestSchedulerDefaultPendingTransferLimit(t *testing.T) { 15 | config := infra.GetTestReflowConfig() 16 | var cluster runner.Cluster 17 | if err := config.Instance(&cluster); err != nil { 18 | t.Fatal(err) 19 | } 20 | scheduler, err := newScheduler(config, log.Std) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | // This is fragile. Due to lack of better ideas, for now... 25 | // TODO(pgopal): may be expose this as an API? 26 | manager, ok := scheduler.Transferer.(*repository.Manager) 27 | if !ok { 28 | t.Fatal("scheduler transferer not repository.Manager") 29 | } 30 | expectedLimit := fmt.Sprintf("limits{def:%d overrides:", defaultTransferLimit) 31 | if !strings.HasPrefix(manager.PendingTransfers.String(), expectedLimit) { 32 | t.Fatalf("expected prefix %s, got %s", expectedLimit, manager.PendingTransfers.String()) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /sched/export_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package sched 6 | 7 | import "github.com/grailbio/reflow" 8 | 9 | func Requirements(tasks []*Task) reflow.Requirements { 10 | return requirements(tasks) 11 | } 12 | 13 | func (t *Task) NonDirectTransfer() bool { 14 | return t.nonDirectTransfer 15 | } 16 | 17 | func (t *Task) WithRepo(repo reflow.Repository) *Task { 18 | t.Repository = repo 19 | return t 20 | } 21 | -------------------------------------------------------------------------------- /syntax/abbrev_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package syntax 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | ) 11 | 12 | func TestAbbrev(t *testing.T) { 13 | for _, c := range []struct{ s, abbrev string }{ 14 | {`"hello world"`, ""}, 15 | {"123456", ""}, 16 | {"[1, 2, 3]", ""}, 17 | {"1 + 2", ""}, 18 | {"((1) + 2)", "1 + 2"}, 19 | {"1 * 2 + 3", "1 * 2 + 3"}, 20 | {"1 * (2 + (3))", "1 * (2 + 3)"}, 21 | {"1*3 + (4 * 4) + 1", "1 * 3 + 4 * 4 + 1"}, 22 | {"1/(3 * 3)", "1 / (3 * 3)"}, 23 | {"1 * (1 + (2 * (3 + 4 * (5 + 6))))", "1 * (1 + 2 * (3 + 4 * (5 + 6)))"}, 24 | {`file("s3://grail-marius/xxx")`, ``}, 25 | {"len( [ 1, 2, 35] )", "len([1, 2, 35])"}, 26 | } { 27 | p := Parser{Mode: ParseExpr, Body: bytes.NewReader([]byte(c.s))} 28 | if err := p.Parse(); err != nil { 29 | t.Errorf("parse error: %v", err) 30 | continue 31 | } 32 | if c.abbrev == "" { 33 | c.abbrev = c.s 34 | } 35 | if got, want := p.Expr.Abbrev(), c.abbrev; got != want { 36 | t.Errorf("for %s, got %v, want %v", p.Expr, got, want) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /syntax/coerce.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package syntax 6 | 7 | import ( 8 | "github.com/grailbio/reflow/errors" 9 | "github.com/grailbio/reflow/flow" 10 | "github.com/grailbio/reflow/internal/scanner" 11 | "github.com/grailbio/reflow/types" 12 | "github.com/grailbio/reflow/values" 13 | ) 14 | 15 | func coerceMatch(v values.T, t *types.T, pos scanner.Position, p Path) (values.T, error) { 16 | if p.Done() { 17 | return v, nil 18 | } 19 | if f, ok := v.(*flow.Flow); ok { 20 | return coerceMatchFlow(f, t, pos, p), nil 21 | } 22 | v, t, p, err := p.Match(v, t) 23 | if err != nil { 24 | return nil, errors.E(pos.String(), err) 25 | } 26 | return coerceMatch(v, t, pos, p) 27 | } 28 | 29 | func coerceMatchFlow(f *flow.Flow, t *types.T, pos scanner.Position, p Path) *flow.Flow { 30 | return &flow.Flow{ 31 | Op: flow.K, 32 | Deps: []*flow.Flow{f}, 33 | FlowDigest: p.Digest(), 34 | K: func(vs []values.T) *flow.Flow { 35 | v, t, p := vs[0], t, p 36 | for { 37 | var err error 38 | v, t, p, err = p.Match(v, t) 39 | if err != nil { 40 | return &flow.Flow{ 41 | Op: flow.Val, 42 | Err: errors.Recover(errors.E(pos.String(), err)), 43 | } 44 | } 45 | if p.Done() { 46 | return toFlow(v, t) 47 | } 48 | if f, ok := v.(*flow.Flow); ok { 49 | return coerceMatchFlow(f, t, pos, p) 50 | } 51 | } 52 | }, 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /syntax/compat.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package syntax 6 | 7 | import ( 8 | "github.com/grailbio/reflow" 9 | "github.com/grailbio/reflow/flow" 10 | "github.com/grailbio/reflow/types" 11 | "github.com/grailbio/reflow/values" 12 | ) 13 | 14 | func fileToFileset(file reflow.File) reflow.Fileset { 15 | return reflow.Fileset{ 16 | Map: map[string]reflow.File{ 17 | ".": file, 18 | }, 19 | } 20 | } 21 | 22 | func dirToFileset(dir values.Dir) reflow.Fileset { 23 | fs := reflow.Fileset{Map: map[string]reflow.File{}} 24 | for scan := dir.Scan(); scan.Scan(); { 25 | fs.Map[scan.Path()] = scan.File() 26 | } 27 | return fs 28 | } 29 | 30 | func coerceToFileset(t *types.T, v values.T) reflow.Fileset { 31 | switch t.Kind { 32 | case types.FileKind: 33 | return fileToFileset(v.(reflow.File)) 34 | case types.DirKind: 35 | return dirToFileset(v.(values.Dir)) 36 | case types.ListKind: 37 | list := v.(values.List) 38 | fs := reflow.Fileset{List: make([]reflow.Fileset, len(list))} 39 | for i := range list { 40 | fs.List[i] = coerceToFileset(t.Elem, list[i]) 41 | } 42 | return fs 43 | default: 44 | panic("invalid input type " + t.String()) 45 | } 46 | } 47 | 48 | var coerceFlowToFilesetDigest = reflow.Digester.FromString("grail.com/reflow/syntax.coerceFlowToFileset") 49 | 50 | func coerceFlowToFileset(t *types.T, f *flow.Flow) *flow.Flow { 51 | return &flow.Flow{ 52 | Op: flow.Coerce, 53 | Deps: []*flow.Flow{f}, 54 | FlowDigest: coerceFlowToFilesetDigest, 55 | Coerce: func(v values.T) (values.T, error) { 56 | return coerceToFileset(t, v), nil 57 | }, 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /syntax/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package syntax 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | 11 | "github.com/grailbio/reflow/internal/scanner" 12 | ) 13 | 14 | // posError attaches a position to an error. 15 | type posError struct { 16 | scanner.Position 17 | err error 18 | } 19 | 20 | func (e posError) Error() string { 21 | return e.Position.String() + ": " + e.err.Error() 22 | } 23 | 24 | // posErrors represents multiple posErrors. 25 | type posErrors []posError 26 | 27 | func (e posErrors) Error() string { 28 | b := new(bytes.Buffer) 29 | for i, err := range e { 30 | b.WriteString(err.Error()) 31 | if i != len(e)-1 { 32 | b.WriteString("\n") 33 | } 34 | } 35 | return b.String() 36 | } 37 | 38 | // errorf formats, and then returns a posErrors. 39 | func errorf(pos scanner.Position, format string, args ...interface{}) error { 40 | return posErrors{{pos, fmt.Errorf(format, args...)}} 41 | } 42 | 43 | // errlist accumulates posErrors. 44 | type errlist []posError 45 | 46 | func (e errlist) Error(pos scanner.Position, err error) errlist { 47 | if err == nil { 48 | return e 49 | } 50 | return append(e, posError{pos, err}) 51 | } 52 | 53 | func (e errlist) Errorf(pos scanner.Position, format string, args ...interface{}) errlist { 54 | return e.Error(pos, fmt.Errorf(format, args...)) 55 | } 56 | 57 | func (e errlist) Append(err error) errlist { 58 | if err == nil { 59 | return e 60 | } 61 | switch err := err.(type) { 62 | case posError: 63 | return append(e, err) 64 | case posErrors: 65 | return append(e, err...) 66 | default: 67 | panic("only posError[s] allowed") 68 | } 69 | } 70 | 71 | func (e errlist) Make() error { 72 | if len(e) > 0 { 73 | return posErrors(e) 74 | } 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /syntax/eval_test/eval_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package eval_test 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | 11 | "github.com/grailbio/reflow/syntax" 12 | "github.com/grailbio/reflow/test/testutil" 13 | ) 14 | 15 | func TestEval(t *testing.T) { 16 | tests := []string{ 17 | "testdata/test1.rf", 18 | "testdata/arith.rf", 19 | "testdata/prec.rf", 20 | "testdata/missingnewline.rf", 21 | "testdata/strings.rf", 22 | "testdata/path.rf", 23 | "testdata/typealias.rf", 24 | "testdata/typealias2.rf", 25 | "testdata/newmodule.rf", 26 | "testdata/delayed.rf", 27 | "testdata/float.rf", 28 | "testdata/regexp.rf", 29 | "testdata/compare.rf", 30 | "testdata/if.rf", 31 | "testdata/dirs.rf", 32 | "testdata/switch.rf", 33 | "testdata/builtin_override.rf", 34 | "testdata/reduce.rf", 35 | "testdata/fold.rf", 36 | "testdata/test_flag_dependence.rf", 37 | "testdata/compr.rf", 38 | "testdata/files.rf", 39 | } 40 | testutil.RunReflowTests(t, tests) 41 | } 42 | 43 | func TestEvalErr(t *testing.T) { 44 | sess := syntax.NewSession(nil) 45 | for _, c := range []struct { 46 | file string 47 | err string 48 | }{ 49 | {"testdata/strings_err1.rf", "number has no digits"}, 50 | {"testdata/strings_err2.rf", "number has no digits"}, 51 | {"testdata/strings_err3.rf", "expected end of string, found '-'"}, 52 | {"testdata/map_compr_err.rf", "failed assertion map_compr_err.TestMapComprErr"}, 53 | {"testdata/list_compr_err.rf", "failed assertion list_compr_err.TestListComprErr"}, 54 | {"testdata/dirs_err.rf", "empty directory testdata/testdir3/"}, 55 | {"testdata/dirs_err2.rf", "dirs.Pick: no files matched *"}, 56 | } { 57 | m, err := sess.Open(c.file) 58 | if err != nil { 59 | t.Errorf("%s: %v", c.file, err) 60 | continue 61 | } 62 | _, err = m.Make(sess, sess.Values) 63 | if err == nil { 64 | t.Errorf("%s: expected error", c.file) 65 | continue 66 | } 67 | if got, want := err.Error(), c.err; !strings.Contains(got, want) { 68 | t.Errorf("%s: got '%v', want '%v'", c.file, got, want) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/arith.rf: -------------------------------------------------------------------------------- 1 | 2 | val test = make("$/test") 3 | 4 | val TestAdd = test.All([ 5 | 1+2 == 3, 6 | 1.0 + 2.0 == 3.0, 7 | ]) 8 | 9 | val TestSub = test.All([ 10 | 10 - 5 == 5, 11 | 10.0 - 5.0 == 5.0, 12 | ]) 13 | 14 | val TestMul = test.All([ 15 | 2*3 == 6, 16 | 2.0*3.0 == 6.0, 17 | ]) 18 | 19 | val TestDiv = test.All([ 20 | 10/2 == 5, 21 | 10.0/2.0 == 5.0, 22 | 11.0/2.0 == 5.5, 23 | ]) 24 | 25 | val TestMod = test.All([ 26 | 10%6 == 4, 27 | ]) 28 | 29 | val TestShift = test.All([ 30 | 1 << 20 == 1048576, 31 | 1048576 >> 20 == 1, 32 | ]) 33 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/builtin_override.rf: -------------------------------------------------------------------------------- 1 | strings := make("$/strings") 2 | val TestDelay = { 3 | delay := func (d bool) => d 4 | delay(true) 5 | } 6 | 7 | val TestFlatten = { 8 | flatten := func (l [[string]]) => [ i | [i] <- l] 9 | flatten([["a"], ["b"]]) == ["a", "b"] 10 | } 11 | 12 | val TestLen = { 13 | len := func(i {size int}) => i.size 14 | len({size: 10}) == 10 15 | } 16 | 17 | val TestList = { 18 | list := func (j [string:int]) => [v | (k, v) <- j] 19 | list(["a": 1]) == [1] 20 | } 21 | 22 | val TestMap = { 23 | map := func (k [string], v [int]) => map([(k1 + strings.FromInt(v1), v1) | k1 <- k, v1 <- v]) 24 | val m1 = map(["a", "b"], [1, 2]) 25 | val m2 = ["a1": 1, "a2": 2, "b1": 1, "b2": 2] 26 | m1 == m2 27 | } 28 | 29 | val TestPanic = { 30 | panic := func (msg string) => "panic: " + msg 31 | panic("test") == "panic: test" 32 | } 33 | 34 | val TestRange = { 35 | range := func (i, j int) => if i == j { [] } else { flatten([[i], range(i+1, j)]) } 36 | val r = range(1, 10) 37 | r == [1, 2, 3, 4, 5, 6, 7, 8, 9] 38 | } 39 | 40 | val TestTrace = { 41 | trace := func (msg string) => "tracing: " + msg 42 | msg := "message" 43 | trace(msg) == "tracing: message" 44 | } 45 | 46 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/delayed.rf: -------------------------------------------------------------------------------- 1 | // This test module tests that the Reflow evaluator handles 2 | // delayed computation properly. 3 | 4 | val TestMap1 = map([(delay("a"), 1), (delay("b"), 2)])["a"] == 1 5 | 6 | val TestMap2 = map([delay(("a", 1)), delay((delay("b"), 2))])["b"] == 2 7 | 8 | val TestApply = { 9 | mul := delay(func(x, y int) => x*y) 10 | val [a, b] = [mul(1, 2), mul(2, 3)] 11 | a == 2 && b == 6 12 | } 13 | 14 | val TestModule = { 15 | mul20 := delay(make("./mul.rf", m := 20)) 16 | mul100 := delay(make("./mul.rf", m := 100)) 17 | val a = delay(mul20.Mul)(10) 18 | val b = delay(mul100.Mul)(10) 19 | a == 200 && b == 1000 20 | } 21 | 22 | val TestUnzip = { 23 | val ([a], [b]) = unzip([delay((1, "ok"))]) 24 | a == 1 && b == "ok" 25 | } 26 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/dirs_err.rf: -------------------------------------------------------------------------------- 1 | val TestEmptyDirErr = dir("testdata/testdir3") 2 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/dirs_err2.rf: -------------------------------------------------------------------------------- 1 | val dirs = make("$/dirs") 2 | 3 | val emptyMap = [:] 4 | val emptyDir = dirs.Make(emptyMap) 5 | val TestEmptyPickErr = dirs.Pick(emptyDir, "*") 6 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/files.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | val files = make("$/files") 3 | 4 | val Test1 = files.Create("hello\n") == file("testdata/testdir2/hello.txt") 5 | val Test2 = files.Create("world\n") == file("testdata/testdir2/world.txt") 6 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/flag_dependence1.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | a string 3 | b string = a + " B" 4 | ) 5 | 6 | val Main = b 7 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/flag_dependence2.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | a string = "C" 3 | b string = a + " B" 4 | ) 5 | 6 | val Main = b 7 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/flag_dependence3.rf: -------------------------------------------------------------------------------- 1 | // params not ascribed to a type 2 | param ( 3 | a = "hello" 4 | b = a + " world" 5 | ) 6 | 7 | val Main = b 8 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/float.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | 3 | val f = 1.232323 4 | 5 | val nf = -f 6 | 7 | val i = int(f) 8 | 9 | val ni = int(nf) 10 | 11 | val TestNegativeUnopFloat = nf == -1.232323 12 | 13 | val TestFloatToIntConv = i == 1 14 | 15 | val TestFloatToIntConvNeg = ni == -1 16 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/fold.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | func add(i, j int) = i + j 3 | 4 | val TestFold = test.All([ 5 | fold(func(i, j int) => i + j, [1, 2, delay(3)], 0) == 6, 6 | fold(func(i, j int) => i - j, [3, 2, delay(1)], 0) == -6, 7 | fold(func(i, j string) => i + j, [delay("ab"), "raca", "dabra"], "") == "abracadabra", 8 | fold(func (i, j float) => i + j, [1.0, 2.1, 3.2], 0.0) == 6.3, 9 | fold(add, [1, 2, 3], 0) == 6, 10 | fold(func(i, j {a int}) => {a: i.a + j.a}, [{a: 1}, {a: 4}, {a: 9}], {a:0}) == {a: 14}, 11 | fold(func (i, j int) => i + j, [], 0) == 0, 12 | fold(func(i, j {a int}) => {a: i.a + j.a}, [{a: 1, b:1}, {a: 4, b:4}, {a: 9, b:9}], {a:0, c:0}) == {a: 14}, 13 | fold(func(i {a, b int}, j {a int}) => {a: i.a + j.a, b: i.b}, [{a: 1, b:1}, {a: 4, b:4}, {a: 9, b:9}], {a:0, b:0, c:2}) == {a: 14, b:0}, 14 | fold(func(i {a int}, j {a int}) => {a: i.a + j.a}, [{a: 1, b:1}, {a: 4, b:4}, {a: 9, b:9}], {a:0, b:0, c:2}) == {a: 14}, 15 | fold(func(i, j {a int}) => {a: i.a + j.a, b: 2}, [{a: 1, b:1}, {a: 4, b:4}, {a: 9, b:9}], {a:0, c:0}) == {a: 14, b:2}, 16 | ]) 17 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/if.rf: -------------------------------------------------------------------------------- 1 | val TestIf1 = if 1 == 1 { 1 } else { 2 } == 1 2 | val TestIf2 = if 1 != 1 { 1 } else { 2 } == 2 3 | val TestIf3 = if 1 != 1 { 1 } else if 3 == 3 { 3 } else { 2 } == 3 4 | val TestIf4 = if 1 != 1 { 1 } else if 3 != 3 { 2 } else if 4 == 4 { 4 } else { 2 } == 4 5 | val TestIf5 = if 1 != 1 { 1 } else if 3 != 3 { 2 } else if 4 != 4 { 4 } else { 2 } == 2 6 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/list_compr_err.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | val TestListComprErr = [test.Assert(bools) | bools <- [[false], [true]]] 3 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/map_compr_err.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | val TestMapComprErr = [test.Assert(bools) | (_, bools) <- ["a": [true], "b": [false]]] 3 | 4 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/missingnewline.rf: -------------------------------------------------------------------------------- 1 | // This module does not terminate with a newline. 2 | 3 | val TestOK = true -------------------------------------------------------------------------------- /syntax/eval_test/testdata/mul.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | l = [1,2,3] 3 | m = 10 4 | ) 5 | 6 | 7 | func Do(op int) = [op*x | x <- l] 8 | 9 | func Mul(op int) = m*op 10 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/newmodule.rf: -------------------------------------------------------------------------------- 1 | 2 | val old = make("./oldmodule.reflow", param1 := "world") 3 | 4 | val TestHello = old.S == "hello world" 5 | 6 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/oldmodule.reflow: -------------------------------------------------------------------------------- 1 | p1 = param("param1", "the first parameter") 2 | 3 | 4 | S = concat("hello ", p1) 5 | 6 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/path.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | val path = make("$/path") 3 | 4 | val url = "s3://foo/bar/baz.ext" 5 | 6 | val TestBase = path.Base(url) == "baz.ext" 7 | val TestExt = path.Ext(url) == ".ext" 8 | val TestDir = path.Dir(url) == "s3://foo/bar" 9 | val TestJoin = path.Join([delay(path.Dir(url)), delay(path.Base(url))]) == url 10 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/prec.rf: -------------------------------------------------------------------------------- 1 | 2 | 3 | func x(y int) = y == 1 4 | 5 | val TestApply = !x(2) 6 | val TestDeref = !{x: x(2)}.x 7 | 8 | val TestArith = 2*3+4 == (2*3)+4 9 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/reduce.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | func add(i, j int) = i + j 3 | 4 | val TestReduce = test.All([ 5 | reduce(func(i, j int) => i + j, [1, 2, delay(3)]) == 6, 6 | reduce(func(i, j int) => i - j, [3, 2, delay(1)]) == 0, 7 | reduce(func(i, j string) => i + j, [delay("ab"), "raca", "dabra"]) == "abracadabra", 8 | reduce(func (i, j float) => i + j, [1.0, 2.1, 3.2]) == 6.3, 9 | reduce(add, [1, 2, 3]) == 6, 10 | reduce(func(i, j {a int}) => {a: i.a + j.a}, [{a: 1}, {a: 4}, {a: 9}]) == {a: 14}, 11 | reduce(func(i, j {a int}) => {a: i.a + j.a}, [{a: 1, c:2}, {a: 4, c: 4}, {a: 9, b: 4}]) == {a: 14}, 12 | reduce(func(i {a, c int}, j {a, c int}) => {a: i.a + j.a, c: i.c}, [{a: 1, c:2}, {a: 4, c: 4}, {a: 9, b: 4, c:2}]) == {a: 14, c: 2}, 13 | ]) 14 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/regexp.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | val regexp = make("$/regexp") 3 | 4 | val TestGroups = { 5 | func eq(x, y [string]) = test.All([x == y | (x, y) <- zip(x, y)]) 6 | eq(regexp.Groups("foo bar", "([[:alpha:]]*) ([[:alpha:]]*)"), ["foo", "bar"]) 7 | } 8 | 9 | val TestReplace = regexp.Replace("foo bar", " (.*)", " baz") == "foo baz" 10 | 11 | val TestMatch = regexp.Match("foo bar", "^foo") && !regexp.Match("blah", "^foo") 12 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/strings_err1.rf: -------------------------------------------------------------------------------- 1 | val strings = make("$/strings") 2 | 3 | val TestToFloatErr = strings.ToFloat("not a valid float") 4 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/strings_err2.rf: -------------------------------------------------------------------------------- 1 | val strings = make("$/strings") 2 | 3 | val TestToFloatErr = strings.ToFloat("1.0e") 4 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/strings_err3.rf: -------------------------------------------------------------------------------- 1 | val strings = make("$/strings") 2 | 3 | val TestToFloatErr = strings.ToFloat("10-10") 4 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/test_flag_dependence.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | val strings = make("$/strings") 3 | 4 | func eq(x, y string) = x == y 5 | 6 | val TestDependence1 = { 7 | case1 := make("./flag_dependence1.rf", a := "A") 8 | case2 := make("./flag_dependence1.rf", a := "A", b := "B") 9 | test.All([ 10 | eq(case1.Main, "A B"), 11 | eq(case2.Main, "B"), 12 | ]) 13 | } 14 | 15 | val TestDependence2 = { 16 | case1 := make("./flag_dependence2.rf", a := "A") 17 | case2 := make("./flag_dependence2.rf", a := "A", b := "B") 18 | case3 := make("./flag_dependence2.rf", b := "B") 19 | case4 := make("./flag_dependence2.rf") 20 | test.All([ 21 | eq(case1.Main, "A B"), 22 | eq(case2.Main, "B"), 23 | eq(case3.Main, "B"), 24 | eq(case4.Main, "C B"), 25 | ]) 26 | } 27 | 28 | val TestDependence3 = { 29 | case1 := make("./flag_dependence3.rf", a := "A") 30 | case2 := make("./flag_dependence3.rf", a := "A", b := "B") 31 | case3 := make("./flag_dependence3.rf", b := "B") 32 | case4 := make("./flag_dependence3.rf") 33 | test.All([ 34 | eq(case1.Main, "A world"), 35 | eq(case2.Main, "B"), 36 | eq(case3.Main, "B"), 37 | eq(case4.Main, "hello world"), 38 | ]) 39 | } 40 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/0 -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/1 -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/2 -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/3 -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/4 -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/5 -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/6 -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/a -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/aa: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/aa -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/aaa: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/aaa -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/aaab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/aaab -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/ab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/ab -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/ac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/ac -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/ad: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/ad -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/b -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/c -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/d -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/e: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/e -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/f -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir/g: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/reflow/90deddd72f8f1b489cab0812e2827c299f77dd19/syntax/eval_test/testdata/testdir/g -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir2/hello.txt: -------------------------------------------------------------------------------- 1 | hello 2 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/testdir2/world.txt: -------------------------------------------------------------------------------- 1 | world 2 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/typealias.rf: -------------------------------------------------------------------------------- 1 | type X {a, b, c int} 2 | 3 | func sel(arg X) = arg.b 4 | 5 | val TestSel = 6 | sel({a: 1, b: 2, c: 3}) == 2 7 | 8 | -------------------------------------------------------------------------------- /syntax/eval_test/testdata/typealias2.rf: -------------------------------------------------------------------------------- 1 | 2 | val mod = make("./typealias.rf") 3 | 4 | type t {x mod.X, y, z string} 5 | 6 | func foo(arg t) = arg.x.a 7 | 8 | val TestFoo = 9 | foo({x: {a: 10, b: 20, c: 30}, y: "ok", z: "blah"}) == 10 10 | -------------------------------------------------------------------------------- /syntax/force_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package syntax 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/grailbio/reflow/types" 11 | "github.com/grailbio/reflow/values" 12 | ) 13 | 14 | func TestForceStruct(t *testing.T) { 15 | typ := types.Struct(&types.Field{Name: "foo", T: types.String}) 16 | v := values.Struct{ 17 | "foo": "vfoo", 18 | "bar": "vbar", 19 | } 20 | vf := Force(v, typ) 21 | s, ok := vf.(values.Struct) 22 | if got, want := ok, true; got != want { 23 | t.Fatalf("got %v, want %v", got, want) 24 | } 25 | if got, want := s["foo"], "vfoo"; got != want { 26 | t.Errorf("got %v, want %v", got, want) 27 | } 28 | } 29 | 30 | func TestForceModule(t *testing.T) { 31 | typ := types.Module( 32 | []*types.Field{{Name: "foo", T: types.String}}, 33 | []*types.Field{}, 34 | ) 35 | v := values.Module{ 36 | "foo": "vfoo", 37 | "bar": "vbar", 38 | } 39 | vf := Force(v, typ) 40 | m, ok := vf.(values.Module) 41 | if got, want := ok, true; got != want { 42 | t.Fatalf("got %v, want %v", got, want) 43 | } 44 | if got, want := m["foo"], "vfoo"; got != want { 45 | t.Errorf("got %v, want %v", got, want) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /syntax/registration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package syntax 6 | 7 | import ( 8 | "io" 9 | 10 | "github.com/grailbio/reflow/types" 11 | ) 12 | 13 | // RegisterModule is a hook to register custom reflow intrinsics. 14 | func RegisterModule(name string, m *ModuleImpl) { 15 | mu.Lock() 16 | lib[name] = m 17 | lib[name].Init(nil, types.NewEnv()) 18 | mu.Unlock() 19 | } 20 | 21 | // ParseAndRegisterModule is like RegisterModule, but parses module from reflow source. 22 | // This function has the advantage of being able to define module parameters. 23 | func ParseAndRegisterModule(name string, source io.Reader) error { 24 | p := Parser{Mode: ParseModule, Body: source} 25 | if err := p.Parse(); err != nil { 26 | return err 27 | } 28 | RegisterModule(name, p.Module) 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /syntax/repeat_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package syntax 6 | 7 | import ( 8 | "fmt" 9 | "math/rand" 10 | "sort" 11 | "testing" 12 | 13 | "github.com/grailbio/base/traverse" 14 | "github.com/grailbio/reflow/flow" 15 | "github.com/grailbio/reflow/types" 16 | "github.com/grailbio/reflow/values" 17 | ) 18 | 19 | const ( 20 | numUniqueValues = 20 21 | numFlows = 500 22 | ) 23 | 24 | func valFlow(v string) *flow.Flow { 25 | return &flow.Flow{Op: flow.Val, FlowDigest: values.Digest(v, types.String), Value: v} 26 | } 27 | 28 | func TestFlowMap(t *testing.T) { 29 | wants := make([]*flow.Flow, numUniqueValues) 30 | flows := make([]*flow.Flow, numFlows) 31 | for i := 0; i < numUniqueValues; i++ { 32 | f := valFlow(fmt.Sprintf("value_%d", i)) 33 | wants[i] = f 34 | flows[i] = f 35 | } 36 | // fill up the rest randomly sampled from the unique set 37 | for i := numUniqueValues; i < numFlows; i++ { 38 | flows[i] = valFlow(wants[rand.Intn(numUniqueValues)].Value.(string)) 39 | } 40 | s := newFlowMap() 41 | _ = traverse.Limit(10).Each(numFlows, func(i int) error { 42 | s.Put(flows[i], flows[i]) 43 | return nil 44 | }) 45 | gots := s.Values() 46 | if got, want := len(gots), numUniqueValues; got != want { 47 | t.Fatalf("got %v, want %v", got, want) 48 | } 49 | sort.Slice(gots, func(i, j int) bool { return gots[i].Value.(string) < gots[j].Value.(string) }) 50 | sort.Slice(wants, func(i, j int) bool { return wants[i].Value.(string) < wants[j].Value.(string) }) 51 | for i := 0; i < numUniqueValues; i++ { 52 | if got, want := gots[i].Digest(), wants[i].Digest(); got != want { 53 | t.Fatalf("got %v, want %v", got, want) 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /syntax/testdata/bundle/concat.rf: -------------------------------------------------------------------------------- 1 | param sep = "" 2 | 3 | func Concat(x, y string) = x+sep+y 4 | -------------------------------------------------------------------------------- /syntax/testdata/bundle/main.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | f1 = "hello" 3 | f2 = "world" 4 | ) 5 | 6 | val concat = make("./concat.rf") 7 | 8 | val Main = concat.Concat(f1, f2) 9 | 10 | -------------------------------------------------------------------------------- /syntax/testdata/bundle/param.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | p1 string 3 | p2 = "ok" 4 | p3 = 123 5 | ) 6 | 7 | val X = p3 8 | val Y = p1+p2 9 | -------------------------------------------------------------------------------- /syntax/testdata/bundle/parammain.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | p1 = "hello" 3 | p3 = 321 4 | ) 5 | 6 | val p = make("p.rfx", p1 := p1, p3 := p3) 7 | val Main = (p.X, p.Y) 8 | -------------------------------------------------------------------------------- /syntax/testdata/constmodule.rf: -------------------------------------------------------------------------------- 1 | param x = 123 2 | 3 | val X = x 4 | val Y = "a truly constant string" 5 | val Z = x*x 6 | -------------------------------------------------------------------------------- /syntax/testdata/err1.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val [x, y, z] = [1, 2] 3 | (x, y, z) 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/err2.rf: -------------------------------------------------------------------------------- 1 | 2 | 3 | val x = panic("panic!") ~> 123 4 | -------------------------------------------------------------------------------- /syntax/testdata/err3.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val [_] = [1, 2] 3 | true 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/err4.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = reduce(func(i, j int) => i + j, []) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/err5.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | 3 | val NoExec = { 4 | val x = 5 + 10 5 | x == 15 6 | } 7 | 8 | val Test = test.Assert([test.ExecRepeatAndCheck(NoExec, 3)]) 9 | -------------------------------------------------------------------------------- /syntax/testdata/err6.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | 3 | val TestAllFail = { 4 | a := 1 5 | b := 2 6 | c := "string" 7 | s := {a, b, c} 8 | test.All([s.a == 1, s.b == 3, s.c == "string"]) 9 | } 10 | -------------------------------------------------------------------------------- /syntax/testdata/err7.rf: -------------------------------------------------------------------------------- 1 | val lib = make("./lib.rf") 2 | 3 | val Test = lib.AssertErr7(1, 2, "string") 4 | -------------------------------------------------------------------------------- /syntax/testdata/err8.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | 3 | val TestAllFail = { 4 | a := 1 5 | b := 2 6 | c := "string" 7 | s := {a, b, c} 8 | test.AssertMap([ 9 | "a": s.a == 0, 10 | "b": s.b == 2, 11 | "c": s.c == "String"]) 12 | } 13 | -------------------------------------------------------------------------------- /syntax/testdata/flag.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | x string 3 | y = file("/dev/null") 4 | z = dir(".") 5 | notFlag = {x: 1, y: 2} 6 | ) 7 | 8 | val X = x 9 | val Y = y 10 | val Z = z 11 | -------------------------------------------------------------------------------- /syntax/testdata/flag1.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | a string = "default" 3 | b string 4 | ) 5 | 6 | val A = a 7 | val B = b 8 | -------------------------------------------------------------------------------- /syntax/testdata/flag_dependence1.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | a string 3 | b string = a + " B" 4 | ) 5 | 6 | val Main = b 7 | -------------------------------------------------------------------------------- /syntax/testdata/flag_dependence2.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | a string = "C" 3 | b string = a + " B" 4 | ) 5 | 6 | val Main = b 7 | -------------------------------------------------------------------------------- /syntax/testdata/flag_dependence3.rf: -------------------------------------------------------------------------------- 1 | // params not ascribed to a type 2 | param ( 3 | a = "hello" 4 | b = a + " world" 5 | ) 6 | 7 | val Main = b 8 | -------------------------------------------------------------------------------- /syntax/testdata/flow.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | a = delay("A") 3 | ) 4 | -------------------------------------------------------------------------------- /syntax/testdata/imagewarn.rf: -------------------------------------------------------------------------------- 1 | param paramImage = "xyz" 2 | 3 | val constImage = "x"+"y"+"z" 4 | 5 | func cat(x, y string) = x+y 6 | 7 | val _ = exec(image := paramImage) (out file) {" "} 8 | val _ = exec(image := paramImage+"x") (out file) {" "} 9 | val _ = exec(image := constImage) (out file) {" "} 10 | val _ = exec(image := "xyz") (out file) {" "} 11 | val _ = exec(image := constImage+"x") (out file) {" "} 12 | val _ = exec(image := cat(constImage, "x")) (out file) {" "} 13 | -------------------------------------------------------------------------------- /syntax/testdata/instantiate.rf: -------------------------------------------------------------------------------- 1 | 2 | val NotConst_1 = make("./constmodule.rf", x := delay(321)).X 3 | val m = make("./constmodule.rf", x := 321) 4 | val Const_2 = m.Y 5 | val Const_3 = m.Z 6 | val Const_3 = make("./constmodule.rf", x := delay(321)).Y 7 | val NotConst_2 = { x := Const_3; x } 8 | -------------------------------------------------------------------------------- /syntax/testdata/lib.rf: -------------------------------------------------------------------------------- 1 | val test = make("$/test") 2 | 3 | func AssertErr7(a, b int, c string) = { 4 | // do 5 | // something 6 | // here 7 | s := {a, b, c} 8 | test.Assert([ 9 | s.a == 1, 10 | s.b == 3, 11 | s.c == "string"]) 12 | } 13 | -------------------------------------------------------------------------------- /syntax/testdata/req.rf: -------------------------------------------------------------------------------- 1 | val mem = 10*GiB 2 | val cpu = 4 3 | val disk = TiB 4 | val cpufeatures = ["intel_avx"] 5 | val nofeatures [string] = [] 6 | 7 | @requires(mem, cpu, disk, cpufeatures, wide := true) 8 | val TestReq1 = file("s3://") 9 | val ExpectReq1 = {mem, cpu: float(cpu), disk, cpufeatures, wide: true} 10 | 11 | @requires(mem := GiB) 12 | val TestReq2 = file("s3://") 13 | val ExpectReq2 = {mem: GiB, cpu: 0.0, disk: 0, cpufeatures: nofeatures, wide: false} 14 | 15 | // cpu can take floats also. it's magic! 16 | @requires(mem, cpu := 0.2) 17 | val TestReq3 = file("s3://") 18 | val ExpectReq3 = {mem, cpu: 0.2, disk: 0, cpufeatures: nofeatures, wide: false} 19 | -------------------------------------------------------------------------------- /syntax/testdata/typerr1.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val (x, y) = (1, 2, 3) 3 | (x, y) 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr10a.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = reduce(func(i, j {a int}) => {c: i.a + j.a}, [{a: 2, b: 3}]) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr10b.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = reduce(func(i {a int}, j {c int}) => {a: i.a + j.c}, [{a: 2, b: 3}]) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr10c.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = reduce(func(i {c int}, j {a int}) => {a: i.c + j.a}, [{a: 2, b: 3}]) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr11.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = reduce(0, [{a: 2, b: 3}]) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr12.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = reduce(func(i {c int}) => {c: i.c}, [{a: 2, b: 3}]) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr13.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = reduce(func(i, j {c int}) => {c: i.c}, {a: 2}) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr14a.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = fold(func(i, j {a int}) => {c: i.a + j.a}, [{a: 2, b: 3}], {a: 2}) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr14b.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = fold(func(i {a int}, j {c int}) => {a: i.a + j.c}, [{a: 2, b: 3}], {a: 2}) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr14c.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = fold(func(i {c int}, j {a int}) => {a: i.c + j.a}, [{a: 2, b: 3}], {a: 2}) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr15.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = fold(0, {a: 2}, 2) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr16.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = fold(func(i {c int}) => {c: i.c}, [{a: 2, b: 3}], {c: 0}) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr17.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = fold(func(i, j {c int}) => {c: i.c}, {a: 2}, {a: 2}) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr18.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | val x = fold(func(i, j {a, b int}) => {a: i.a + j.a, b: i.b + j.b}, [{a: 2}], {a: 2}) 3 | x 4 | } 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr19.rf: -------------------------------------------------------------------------------- 1 | func TestExec(in file) = 2 | exec(image := "ubuntu", nondeterministic := 1.2) (out file) {" 3 | cat {{in}} > {{out}} 4 | "} 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr2.rf: -------------------------------------------------------------------------------- 1 | val Test = { 2 | a := 1 3 | b := 2 4 | c := 3 5 | [(x, y) | (x, y) <- {a, b, c}] 6 | } 7 | -------------------------------------------------------------------------------- /syntax/testdata/typerr20.rf: -------------------------------------------------------------------------------- 1 | val Test = error("hello", "world") -------------------------------------------------------------------------------- /syntax/testdata/typerr21.rf: -------------------------------------------------------------------------------- 1 | val x = 42 2 | val Test = error(x, x) 3 | -------------------------------------------------------------------------------- /syntax/testdata/typerr3.rf: -------------------------------------------------------------------------------- 1 | 2 | func F(x, y, z string) = x 3 | 4 | val Test = F("", file(""), "") 5 | -------------------------------------------------------------------------------- /syntax/testdata/typerr4.rf: -------------------------------------------------------------------------------- 1 | // Regression test for type environment leaking across recursive module 2 | // instantiations. 3 | 4 | val x = 123 5 | val Test = make("./typerr4mod.rf") 6 | 7 | -------------------------------------------------------------------------------- /syntax/testdata/typerr4mod.rf: -------------------------------------------------------------------------------- 1 | val X = x 2 | -------------------------------------------------------------------------------- /syntax/testdata/typerr5.reflow: -------------------------------------------------------------------------------- 1 | P = param("invalid-parameter-name", "") 2 | -------------------------------------------------------------------------------- /syntax/testdata/typerr5.rf: -------------------------------------------------------------------------------- 1 | val Test = make("./typerr5.reflow") 2 | -------------------------------------------------------------------------------- /syntax/testdata/typerr6.rf: -------------------------------------------------------------------------------- 1 | @requires(cpu := len(file("x"))) 2 | val Main = "x" 3 | -------------------------------------------------------------------------------- /syntax/testdata/typerr7.rf: -------------------------------------------------------------------------------- 1 | 2 | // delayed image is not ok, but cpu is 3 | val Main = exec(image := delay("x"), cpu := delay(1)) (out file) {" "} 4 | -------------------------------------------------------------------------------- /syntax/testdata/typerr8.rf: -------------------------------------------------------------------------------- 1 | val Test = switch "a" { 2 | case (a, b): true 3 | } 4 | -------------------------------------------------------------------------------- /syntax/testdata/typerr9.rf: -------------------------------------------------------------------------------- 1 | val Test = switch ["a"] { 2 | case [a, b]: true 3 | } 4 | -------------------------------------------------------------------------------- /syntax/testdata/unusedwarn.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | x int 3 | y = "ok" 4 | ) 5 | 6 | z := 123 7 | Z := "ok" 8 | 9 | func x(x, y, z string) = 123 10 | func X(x, y, z string) = x 11 | 12 | func Y(x, y, z string) = { 13 | blah := x 14 | z 15 | } 16 | 17 | val Compr = [1 | x <- [1,2,3]] 18 | 19 | -------------------------------------------------------------------------------- /taskdb/noptaskdb/noptaskdb.go: -------------------------------------------------------------------------------- 1 | package noptaskdb 2 | 3 | import ( 4 | "github.com/grailbio/infra" 5 | "github.com/grailbio/reflow/repository/noprepo" 6 | "github.com/grailbio/reflow/taskdb" 7 | "github.com/grailbio/reflow/test/testutil" 8 | ) 9 | 10 | func init() { 11 | infra.Register("noptaskdb", new(NopTaskDB)) 12 | } 13 | 14 | // NopTaskDB is a nop-TaskDB infra provider 15 | type NopTaskDB struct { 16 | taskdb.TaskDB 17 | } 18 | 19 | // Init implements infra.Provider 20 | func (t *NopTaskDB) Init() error { 21 | t.TaskDB = testutil.NewNopTaskDB(&noprepo.Repository{}) 22 | return nil 23 | } 24 | -------------------------------------------------------------------------------- /test/infra/test.go: -------------------------------------------------------------------------------- 1 | package infra 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws/session" 5 | "github.com/grailbio/infra" 6 | _ "github.com/grailbio/infra/aws/test" 7 | "github.com/grailbio/infra/tls" 8 | "github.com/grailbio/reflow" 9 | "github.com/grailbio/reflow/assoc" 10 | _ "github.com/grailbio/reflow/assoc/test" 11 | _ "github.com/grailbio/reflow/ec2cluster/test" 12 | infra2 "github.com/grailbio/reflow/infra" 13 | "github.com/grailbio/reflow/log" 14 | _ "github.com/grailbio/reflow/repository/s3/test" 15 | "github.com/grailbio/reflow/runner" 16 | ) 17 | 18 | // GetTestReflowConfig returns a dummy reflow config. 19 | func GetTestReflowConfig() infra.Config { 20 | schema := infra.Schema{ 21 | infra2.Log: new(log.Logger), 22 | infra2.Repository: new(reflow.Repository), 23 | infra2.Cluster: new(runner.Cluster), 24 | infra2.Assoc: new(assoc.Assoc), 25 | infra2.Cache: new(infra2.CacheProvider), 26 | infra2.Session: new(session.Session), 27 | infra2.TLS: new(tls.Certs), 28 | } 29 | keys := GetTestReflowConfigKeys() 30 | cfg, err := schema.Make(keys) 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | return cfg 35 | } 36 | 37 | // GetTestReflowConfigKeys returns a dummy set of infra keys. 38 | func GetTestReflowConfigKeys() infra.Keys { 39 | return infra.Keys{ 40 | infra2.Assoc: "fakeassoc", 41 | infra2.Cache: "readwrite", 42 | infra2.Cluster: "fakecluster", 43 | infra2.Log: "logger,level=debug", 44 | infra2.Repository: "fakes3", 45 | infra2.Session: "fakesession", 46 | infra2.TLS: "tls,file=/tmp/ca.reflow", 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /test/regress/fake.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package regress 6 | 7 | // Phabricator requires that each directory with .go files 8 | // is a buildable Go package. 9 | // 10 | // So my only purpose is to be an empty Go file. 11 | // 12 | // How's your day going? 13 | -------------------------------------------------------------------------------- /test/regress/regress_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !unit regress 6 | 7 | package regress 8 | 9 | import ( 10 | "flag" 11 | "io/ioutil" 12 | "os" 13 | "os/exec" 14 | "path/filepath" 15 | "testing" 16 | ) 17 | 18 | var ( 19 | binary = flag.String("regress_test.binary", "", "reflow binary to use for the test") 20 | localDir = flag.String("regress_test.localdir", "", "local dir to use for the test") 21 | ) 22 | 23 | // TestRegress performs regression checking, and requires AWS credentials for file transfers. 24 | func TestRegress(t *testing.T) { 25 | if *binary == "" { 26 | const reflow = "./test.reflow" 27 | cmd := exec.Command("go", "build", "-o", reflow, "github.com/grailbio/reflow/cmd/reflow") 28 | if err := cmd.Run(); err != nil { 29 | t.Fatalf("go build: %s", err) 30 | } 31 | defer os.Remove(reflow) 32 | *binary = reflow 33 | } 34 | tests, err := ioutil.ReadDir("testdata") 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | args := []string{"-log", "debug", "run", "-local"} 39 | if "" != *localDir { 40 | args = append(args, "-localdir", *localDir) 41 | } 42 | for _, test := range tests { 43 | t.Run(test.Name(), func(t *testing.T) { 44 | if filepath.Ext(test.Name()) != ".rf" || filepath.Base(test.Name()) == "generate.rf" { 45 | return 46 | } 47 | t.Parallel() 48 | testargs := append(args, filepath.Join("testdata", test.Name())) 49 | // Run using (reflow) cache. 50 | cmd := exec.Command(*binary, testargs...) 51 | if out, err := cmd.CombinedOutput(); err != nil { 52 | t.Errorf("%s: %s\n%s", test.Name(), err, string(out)) 53 | } 54 | 55 | // Run without (reflow) cache. 56 | cmd = exec.Command(*binary, append([]string{"-cache", "off"}, testargs...)...) 57 | if out, err := cmd.CombinedOutput(); err != nil { 58 | t.Errorf("%s (-cache=off): %s\n%s", test.Name(), err, string(out)) 59 | } 60 | }) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /test/regress/testdata/bridge.rf: -------------------------------------------------------------------------------- 1 | // Tests v0->v1 bridging. 2 | val gen = make("./generate.rf") 3 | val test = make("$/test") 4 | 5 | @requires(cpu := 1) 6 | val Main = gen.GenerateFixed("localfile://testdata/inputdata") ~> { 7 | filesets := make("$/filesets") 8 | files := make("$/files") 9 | old := make("./oldmodule.reflow", f := "localfile://testdata/inputdata") 10 | f := file("testdata/inputdata") 11 | test.Assert([len(files.Fileset(old.X)) == len(f)]) 12 | } 13 | -------------------------------------------------------------------------------- /test/regress/testdata/coerce.rf: -------------------------------------------------------------------------------- 1 | // This tests that match coercion works on multiple levels of lazily computed data. 2 | val gen = make("./generate.rf") 3 | 4 | val f2 = gen.GenerateRand("localfile://testdata/input1") ~> 5 | gen.GenerateRand("localfile://testdata/input2") ~> 6 | file("localfile://testdata/input1") 7 | 8 | val fs = f2 ~> [ 9 | "x": (123, file("localfile://testdata/input2")), 10 | ] 11 | 12 | @requires(cpu := 1) 13 | val Main = { 14 | val (_, f) = fs["x"] 15 | len(f) 16 | } 17 | -------------------------------------------------------------------------------- /test/regress/testdata/flagparam.rf: -------------------------------------------------------------------------------- 1 | // This tests that flag parameters can depend on each other. 2 | 3 | param ( 4 | x = "ok" 5 | y = x+"!" 6 | ) 7 | 8 | val test = make("$/test") 9 | 10 | val Main = test.Assert([ 11 | y == "ok!", 12 | ]) 13 | -------------------------------------------------------------------------------- /test/regress/testdata/generate.rf: -------------------------------------------------------------------------------- 1 | val files = make("$/files") 2 | 3 | @requires(cpu := 1) 4 | func GenerateRand(path string) = { 5 | val f = exec(image := "ubuntu") (out file) {" 6 | head -c 1K {{out}} 7 | "} 8 | files.Copy(f, path) 9 | } 10 | 11 | @requires(cpu := 1) 12 | func GenerateFixed(path string) = { 13 | inputdata := file("testdata/inputdata") 14 | val f = exec(image := "ubuntu") (out file) {" 15 | cat {{inputdata}} > {{out}} 16 | "} 17 | files.Copy(f, path) 18 | } 19 | -------------------------------------------------------------------------------- /test/regress/testdata/inputdata: -------------------------------------------------------------------------------- 1 | This is a sample file with the same content used for the purpose of testing -------------------------------------------------------------------------------- /test/regress/testdata/mapdigest.rf: -------------------------------------------------------------------------------- 1 | // This tests that digests are properly computed on lazy map keys. 2 | 3 | val dirs = make("$/dirs") 4 | val regexp = make("$/regexp") 5 | val strings = make("$/strings") 6 | 7 | val key = file("/dev/null") ~> "ok" 8 | 9 | func x(m [string:int]) = m["a"] 10 | 11 | @requires(cpu := 1) 12 | val Main = x([ 13 | "a": 333, 14 | ...map([(key, 123)]), 15 | ]) 16 | -------------------------------------------------------------------------------- /test/regress/testdata/oldmodule.reflow: -------------------------------------------------------------------------------- 1 | 2 | X = intern(param("f", "")) 3 | 4 | -------------------------------------------------------------------------------- /test/regress/testdata/repeatexec.rf: -------------------------------------------------------------------------------- 1 | // This tests test.ExecRepeatAndCheck functionality. 2 | 3 | val gen = make("./generate.rf") 4 | val test = make("$/test") 5 | 6 | func CatCopy(in file, random bool) = 7 | if random { 8 | exec(image := "ubuntu", mem := 10*MiB) (out file) {" 9 | echo $RANDOM > {{out}} 10 | cat {{in}} >> {{out}} 11 | "} 12 | } else { 13 | exec(image := "ubuntu", mem := 10*MiB) (out file) {" 14 | cat {{in}} > {{out}} 15 | "} 16 | } 17 | 18 | @requires(cpu := 1) 19 | val Main = gen.GenerateFixed("localfile://inputdata") ~> test.Assert([ 20 | test.ExecRepeatAndCheck(CatCopy(file("localfile://inputdata"), false), 1), 21 | test.ExecRepeatAndCheck(CatCopy(file("localfile://inputdata"), false), 3), 22 | false == test.ExecRepeatAndCheck(CatCopy(file("localfile://inputdata"), true), 3), 23 | ]) 24 | -------------------------------------------------------------------------------- /test/regress/testdata/repeatexecskip.rf: -------------------------------------------------------------------------------- 1 | // This tests test.ExecRepeatAndCheck functionality. 2 | 3 | val gen = make("./generate.rf") 4 | val test = make("$/test") 5 | 6 | func CatCopyRand(in file) = 7 | exec(image := "ubuntu", mem := 10*MiB, nondeterministic := true) (out file) {" 8 | echo $RANDOM > {{out}} 9 | cat {{in}} >> {{out}} 10 | "} 11 | 12 | @requires(cpu := 1) 13 | val Main = gen.GenerateRand("localfile://inputdata") ~> test.Assert([ 14 | test.ExecRepeatAndCheck(CatCopyRand(file("localfile://inputdata")), 3), 15 | ]) 16 | -------------------------------------------------------------------------------- /test/testutil/reflowtestrunner_test.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestRunReflowTests(t *testing.T) { 10 | rf, err := ioutil.TempFile("", "reflowtestrunner_*.rf") 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | duplicateTestNames := `val TestFoo = false 15 | val TestFoo = true` 16 | _, err = rf.WriteString(duplicateTestNames) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | 21 | defer func() { 22 | r := recover() 23 | if r == nil { 24 | t.Fatalf("RunReflowTests should have panicked due to reuse of TestFoo") 25 | } 26 | errstr := r.(string) 27 | if !strings.Contains(errstr, "Test name 'TestFoo' is used more than once") { 28 | t.Errorf("RunReflowTests panicked, but not because of the reused test name: %s", errstr) 29 | } 30 | }() 31 | RunReflowTests(t, []string{rf.Name()}) 32 | } 33 | -------------------------------------------------------------------------------- /test/testutil/repositorycallkind_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=RepositoryCallKind"; DO NOT EDIT 2 | 3 | package testutil 4 | 5 | import "fmt" 6 | 7 | const _RepositoryCallKind_name = "RepositoryGetRepositoryPutRepositoryStatRepositoryWriteToRepositoryReadFrom" 8 | 9 | var _RepositoryCallKind_index = [...]uint8{0, 13, 26, 40, 57, 75} 10 | 11 | func (i RepositoryCallKind) String() string { 12 | if i < 0 || i >= RepositoryCallKind(len(_RepositoryCallKind_index)-1) { 13 | return fmt.Sprintf("RepositoryCallKind(%d)", i) 14 | } 15 | return _RepositoryCallKind_name[_RepositoryCallKind_index[i]:_RepositoryCallKind_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /test/testutil/skipifnocreds.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aws/aws-sdk-go/aws/awserr" 7 | "github.com/aws/aws-sdk-go/aws/credentials" 8 | ) 9 | 10 | // SkipIfNoCreds allows a test to be skipped if no credentials are found. 11 | // Caution: Renaming/removing this will prevent execution of tests (with credentials) which call this. 12 | func SkipIfNoCreds(t *testing.T) { 13 | t.Helper() 14 | provider := &credentials.ChainProvider{ 15 | VerboseErrors: true, 16 | Providers: []credentials.Provider{ 17 | &credentials.EnvProvider{}, 18 | &credentials.SharedCredentialsProvider{}, 19 | }, 20 | } 21 | _, err := provider.Retrieve() 22 | if err != nil { 23 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NoCredentialProviders" { 24 | t.Skip("no credentials in environment; skipping") 25 | } 26 | t.Fatal(err) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test/testutil/testutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package testutil contains various utilities for testing Reflow functionality. 6 | package testutil 7 | -------------------------------------------------------------------------------- /tool/batch_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "testing" 14 | 15 | "github.com/grailbio/testutil" 16 | ) 17 | 18 | func TestBatch(t *testing.T) { 19 | dir, cleanup := testutil.TempDir(t, "", "") 20 | defer cleanup() 21 | var ( 22 | cmd Cmd 23 | ctx = context.Background() 24 | ) 25 | saveDir, err := os.Getwd() 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | defer func() { 30 | if err := os.Chdir(saveDir); err != nil { 31 | t.Fatal(err) 32 | } 33 | }() 34 | if err := os.Chdir("testdata"); err != nil { 35 | t.Fatal(err) 36 | } 37 | filename := filepath.Join(dir, "batch.rf") 38 | cmd.genbatch(ctx, "-o", filename) 39 | p, err := ioutil.ReadFile(filename) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | if got, want := string(p), `val Main = [ 44 | make("test.rf", 45 | b := true, 46 | d := dir("s3://grail-marius/1.dir/"), 47 | f := file("s3://grail-marius/1.file"), 48 | fl := 1.1, 49 | i := 1, 50 | s := "a", 51 | ).Main, 52 | make("test.rf", 53 | b := false, 54 | d := dir("s3://grail-marius/2.dir/"), 55 | f := file("s3://grail-marius/2.file"), 56 | fl := 2.2, 57 | i := 2, 58 | s := "b", 59 | ).Main, 60 | ] 61 | `; !strings.HasSuffix(got, want) { 62 | t.Errorf("got %v, want %v", got, want) 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /tool/bundle.go: -------------------------------------------------------------------------------- 1 | package tool 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/grailbio/reflow/syntax" 10 | ) 11 | 12 | func (c *Cmd) bundle(ctx context.Context, args ...string) { 13 | flags := flag.NewFlagSet("bundle", flag.ExitOnError) 14 | out := flags.String("o", "", "output path of bundle") 15 | nowarn := flags.Bool("nowarn", false, "suppress warnings") 16 | help := `Bundle type checks, then writes a self-contained Reflow bundle to a 17 | file named by the basename of the provided Reflow module with the 18 | suffix ".rfx". 19 | 20 | Reflow bundles are interchangeable with other Reflow modules: they 21 | may be run with command run or imported by other Reflow modules. Any 22 | flags provided as arguments provide default values to the module's 23 | parameters.` 24 | c.Parse(flags, args, help, "bundle [-o output] path [args]") 25 | if flags.NArg() == 0 { 26 | flags.Usage() 27 | } 28 | file, args := flags.Arg(0), flags.Args()[1:] 29 | if ext := filepath.Ext(file); ext != ".rf" { 30 | c.Fatalf("extension %s not supported for bundling", ext) 31 | } 32 | sess := syntax.NewSession(nil) 33 | if !*nowarn { 34 | sess.Stdwarn = c.Stderr 35 | } 36 | m, err := sess.Open(file) 37 | c.must(err) 38 | c.must(m.InjectArgs(sess, args)) 39 | if *out == "" { 40 | *out = filepath.Base(file) + "x" // ".rfx" 41 | } 42 | f, err := os.Create(*out) 43 | c.must(err) 44 | c.must(sess.Bundle().WriteTo(f)) 45 | c.must(f.Close()) 46 | } 47 | -------------------------------------------------------------------------------- /tool/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "bufio" 9 | "context" 10 | "flag" 11 | "os" 12 | 13 | "github.com/grailbio/reflow" 14 | "github.com/grailbio/reflow/assoc" 15 | ) 16 | 17 | func (c *Cmd) rmcache(ctx context.Context, args ...string) { 18 | var ( 19 | flags = flag.NewFlagSet("rmcache", flag.ExitOnError) 20 | help = `Rmcache removes items from cache. 21 | Items are digests read from the standard input.` 22 | ) 23 | c.Parse(flags, args, help, "rmcache") 24 | if flags.NArg() != 0 { 25 | flags.Usage() 26 | } 27 | 28 | var ass assoc.Assoc 29 | c.must(c.Config.Instance(&ass)) 30 | 31 | var n int 32 | scan := bufio.NewScanner(os.Stdin) 33 | for scan.Scan() { 34 | id, err := reflow.Digester.Parse(scan.Text()) 35 | if err != nil { 36 | c.Log.Errorf("failed to parse %s: %v; skipping", scan.Text(), err) 37 | continue 38 | } 39 | // TODO(marius): parallelize this for large jobs. 40 | if err := ass.Delete(ctx, id); err != nil { 41 | c.Log.Errorf("failed to delete %s: %v", id, err) 42 | } 43 | c.Log.Debugf("removed key %v", id) 44 | n++ 45 | } 46 | c.Log.Debugf("removed %d keys", n) 47 | } 48 | -------------------------------------------------------------------------------- /tool/cat.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "io" 11 | 12 | "github.com/grailbio/base/digest" 13 | "github.com/grailbio/reflow" 14 | "github.com/grailbio/reflow/errors" 15 | "github.com/grailbio/reflow/taskdb" 16 | ) 17 | 18 | func (c *Cmd) cat(ctx context.Context, args ...string) { 19 | flags := flag.NewFlagSet("cat", flag.ExitOnError) 20 | taskdbRepoFlag := flags.Bool("taskdbrepo", false, "whether to use the taskdb repository instead of reflow repository") 21 | help := `Cat copies files from Reflow's repository (by default) 22 | If the flag -taskdbrepo is provided, then Cat will use the configured TaskDB repository 23 | ` 24 | c.Parse(flags, args, help, "cat [-taskdbrepo] files...") 25 | if flags.NArg() == 0 { 26 | flags.Usage() 27 | } 28 | var repo reflow.Repository 29 | if *taskdbRepoFlag { 30 | var tdb taskdb.TaskDB 31 | c.must(c.Config.Instance(&tdb)) 32 | repo = tdb.Repository() 33 | } else { 34 | c.must(c.Config.Instance(&repo)) 35 | } 36 | var ids []digest.Digest 37 | for _, arg := range flags.Args() { 38 | id, err := reflow.Digester.Parse(arg) 39 | if err != nil { 40 | c.Fatalf("parse %s: %v", id, err) 41 | } 42 | ids = append(ids, id) 43 | } 44 | for _, id := range ids { 45 | _, err := repo.Stat(ctx, id) 46 | if err != nil { 47 | // TODO(swami): Should we just lookup all relevant repos instead of forcing the user to specify ? 48 | if errors.Is(errors.NotExist, err) { 49 | c.Fatalf("stat %s: %v\nDo you need to specify -taskdbrepo (you would if the reference was from taskdb)", id.Hex(), err) 50 | } 51 | c.Fatalf("stat %s: %v", id.Hex(), err) 52 | } 53 | } 54 | for _, id := range ids { 55 | rc, err := repo.Get(ctx, id) 56 | if err != nil { 57 | c.Fatalf("get %s: %v", id, err) 58 | } 59 | _, err = io.Copy(c.Stdout, rc) 60 | if err != nil { 61 | c.Fatalf("read %s: %v", id, err) 62 | } 63 | rc.Close() 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tool/check.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | package tool 5 | 6 | import ( 7 | "context" 8 | "flag" 9 | 10 | "github.com/grailbio/reflow/syntax" 11 | ) 12 | 13 | func (c *Cmd) check(ctx context.Context, args ...string) { 14 | flags := flag.NewFlagSet("check", flag.ExitOnError) 15 | warnErr := flags.Bool("w", false, "treat warnings as errors") 16 | help := `Check typechecks the provided modules and prints typechecking 17 | warnings. If any errors are encountered during typechecking, check 18 | exits with code 1.` 19 | c.Parse(flags, args, help, "check modules...") 20 | if flags.NArg() == 0 { 21 | flags.Usage() 22 | } 23 | sess := syntax.NewSession(nil) 24 | sess.Stdwarn = c.Stderr 25 | ok := true 26 | for i := 0; i < flags.NArg(); i++ { 27 | if _, err := sess.Open(flags.Arg(i)); err != nil { 28 | c.Errorln(err) 29 | ok = false 30 | } 31 | } 32 | if !ok || *warnErr && sess.NWarn() > 0 { 33 | c.Exit(1) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tool/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "os" 11 | "runtime" 12 | ) 13 | 14 | // Parse parses the provided FlagSet from the provided arguments. It 15 | // adds a -help flag to the flagset, and prints the help and usage 16 | // string when the command is called with -help. On usage error, it 17 | // prints the flag defaults and exits with code 2. 18 | func (c *Cmd) Parse(fs *flag.FlagSet, args []string, help, usage string) { 19 | helpFlag := fs.Bool("help", false, "display subcommand help") 20 | fs.Usage = func() { 21 | fmt.Fprintln(os.Stderr, "usage: reflow "+usage) 22 | fmt.Fprintln(os.Stderr, "Flags:") 23 | fs.PrintDefaults() 24 | c.Exit(2) 25 | } 26 | 27 | if err := fs.Parse(args); err != nil { 28 | c.Fatal(err) 29 | } 30 | if *helpFlag { 31 | fmt.Fprintln(os.Stderr, "usage: reflow "+usage) 32 | fmt.Fprintln(os.Stderr) 33 | fmt.Fprintln(os.Stderr, help) 34 | fmt.Fprintln(os.Stderr) 35 | fmt.Fprintln(os.Stderr, "Flags:") 36 | fs.PrintDefaults() 37 | c.Exit(0) 38 | } 39 | } 40 | 41 | func (c Cmd) must(err error) { 42 | if err != nil { 43 | _, file, line, _ := runtime.Caller(1) 44 | c.Fatalf("%s:%d: %v", file, line, err) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /tool/cost_test.go: -------------------------------------------------------------------------------- 1 | package tool 2 | 3 | import "testing" 4 | 5 | func TestCost(t *testing.T) { 6 | var nilCost Cost 7 | for _, tt := range []struct { 8 | a, b, want Cost 9 | m float64 10 | }{ 11 | {NewCostExact(1.0), nilCost, NewCostExact(1.0), 1}, 12 | {nilCost, NewCostExact(1.0), NewCostExact(1.0), 1}, 13 | {NewCostExact(1.0), NewCostUB(0), NewCostUB(1.0), 1}, 14 | {NewCostUB(0.0), NewCostExact(1.0), NewCostUB(1.0), 1}, 15 | {NewCostExact(1.0), NewCostExact(2.5), NewCostExact(3.5), 1}, 16 | {NewCostUB(1.0), NewCostExact(2.5), NewCostUB(3.5), 1}, 17 | {NewCostExact(1.0), NewCostUB(2.5), NewCostUB(3.5), 1}, 18 | {NewCostUB(1.0), NewCostUB(2.5), NewCostUB(3.5), 1}, 19 | {NewCostExact(1.0), NewCostExact(2.5), NewCostExact(1.75), 0.5}, 20 | {NewCostUB(1.0), NewCostExact(2.5), NewCostUB(1.75), 0.5}, 21 | {NewCostExact(1.0), NewCostUB(2.5), NewCostUB(1.75), 0.5}, 22 | {NewCostUB(1.0), NewCostUB(2.5), NewCostUB(1.75), 0.5}, 23 | } { 24 | v := tt.a 25 | v.Add(tt.b) 26 | v.Mul(tt.m) 27 | if got, want := v, tt.want; got != want { 28 | t.Errorf("got %s, want %s", got, want) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tool/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "io" 11 | "sort" 12 | 13 | "github.com/grailbio/reflow/syntax" 14 | "v.io/x/lib/textutil" 15 | ) 16 | 17 | func (c *Cmd) doc(ctx context.Context, args ...string) { 18 | flags := flag.NewFlagSet("doc", flag.ExitOnError) 19 | help := "Doc displays documentation for Reflow modules." 20 | c.Parse(flags, args, help, "doc path") 21 | 22 | if flags.NArg() == 0 { 23 | c.Println("Reflow's system modules are:") 24 | names := syntax.Modules() 25 | sort.Strings(names) 26 | for _, name := range names { 27 | c.Printf(" $/%s\n", name) 28 | } 29 | return 30 | } 31 | if flags.NArg() != 1 { 32 | flags.Usage() 33 | } 34 | sess := syntax.NewSession(nil) 35 | m, err := sess.Open(flags.Arg(0)) 36 | c.must(err) 37 | if params := m.Params(); len(params) > 0 { 38 | c.Println("Parameters") 39 | c.Println() 40 | for _, p := range params { 41 | if p.Required { 42 | c.Printf("val %s %s (required)\n", p.Ident, p.Type) 43 | } else { 44 | c.Printf("val %s %s = %s\n", p.Ident, p.Type, p.Expr.Abbrev()) 45 | } 46 | c.printdoc(p.Doc, "") 47 | } 48 | c.Println() 49 | } 50 | 51 | c.Println("Declarations") 52 | c.Println() 53 | for _, f := range m.Type(nil).Aliases { 54 | c.Printf("type %s %s\n", f.Name, f.T) 55 | c.printdoc(m.Doc(f.Name), "\n") 56 | } 57 | for _, f := range m.Type(nil).Fields { 58 | c.Printf("val %s %s\n", f.Name, f.T) 59 | c.printdoc(m.Doc(f.Name), "\n") 60 | } 61 | } 62 | 63 | func (c *Cmd) printdoc(doc string, nl string) { 64 | if doc == "" { 65 | c.Printf("%s", nl) 66 | return 67 | } 68 | pw := textutil.PrefixLineWriter(c.Stdout, " ") 69 | ww := textutil.NewUTF8WrapWriter(pw, 80) 70 | if _, err := io.WriteString(ww, doc); err != nil { 71 | c.Fatal(err) 72 | } 73 | ww.Flush() 74 | pw.Flush() 75 | c.Printf("%s", nl) 76 | } 77 | -------------------------------------------------------------------------------- /tool/ec2verify_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | 11 | "github.com/grailbio/reflow/ec2cluster/instances" 12 | ) 13 | 14 | func TestFilterInstanceTypes(t *testing.T) { 15 | instanceTypes := []instances.Type{ 16 | {Name: "a"}, 17 | {Name: "b"}, 18 | {Name: "c"}, 19 | {Name: "d"}, 20 | } 21 | existing := map[string]instances.VerifiedStatus{ 22 | "a": {Attempted: true, Verified: true, ApproxETASeconds: 10, MemoryBytes: 0}, 23 | "b": {Attempted: true, Verified: false, ApproxETASeconds: 70, MemoryBytes: 0}, 24 | "c": {Attempted: false, Verified: false, ApproxETASeconds: -1, MemoryBytes: 0}, 25 | } 26 | for _, tt := range []struct { 27 | instanceTypes []instances.Type 28 | existing map[string]instances.VerifiedStatus 29 | retry bool 30 | toverify []string 31 | }{ 32 | {instanceTypes, map[string]instances.VerifiedStatus{}, false, []string{"a", "b", "c", "d"}}, 33 | {instanceTypes, existing, false, []string{"c", "d"}}, 34 | {instanceTypes, existing, true, []string{"b", "c", "d"}}, 35 | } { 36 | toverify := instancesToVerify(tt.instanceTypes, tt.existing, tt.retry) 37 | if got, want := toverify, tt.toverify; !reflect.DeepEqual(got, want) { 38 | t.Errorf("got %v want %v", got, want) 39 | } 40 | } 41 | } 42 | 43 | func TestExpectedMemoryBytes(t *testing.T) { 44 | for _, tt := range []struct { 45 | instanceType string 46 | expectedMem int64 47 | }{ 48 | {"c3.large", 3654045523}, 49 | {"m5.large", 7563137260}, 50 | {"c5.4xlarge", 30567888547}, 51 | {"m5dn.xlarge", 15341268848}, 52 | } { 53 | if got, want := instances.VerifiedByRegion["us-west-2"][tt.instanceType].ExpectedMemoryBytes(), tt.expectedMem; got != want { 54 | t.Errorf("%s: got %d, want %d", tt.instanceType, got, want) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /tool/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "io" 11 | 12 | "github.com/grailbio/reflow/runtime" 13 | ) 14 | 15 | func (c *Cmd) http(ctx context.Context, args ...string) { 16 | var ( 17 | flags = flag.NewFlagSet("http", flag.ExitOnError) 18 | help = `Command http can be used to do an HTTP GET on any URL. For example: 19 | reflow http https://:9000/debug/vars - will dump the vars from the reflowlet 20 | reflow http https://:9000/debug/pprof - to see profiling data` 21 | ) 22 | c.Parse(flags, args, help, "http url") 23 | if flags.NArg() != 1 { 24 | flags.Usage() 25 | } 26 | arg := flags.Arg(0) 27 | 28 | httpClient, err := runtime.HttpClient(c.Config) 29 | c.must(err) 30 | resp, err := httpClient.Get(arg) 31 | c.must(err) 32 | if resp.Body == nil { 33 | return 34 | } 35 | defer resp.Body.Close() 36 | if _, err := io.Copy(c.Stdout, resp.Body); err != nil { 37 | c.Fatal(err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tool/images.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | "os" 12 | "sort" 13 | 14 | "github.com/grailbio/reflow/syntax" 15 | "github.com/grailbio/reflow/types" 16 | ) 17 | 18 | func (c *Cmd) images(ctx context.Context, args ...string) { 19 | flags := flag.NewFlagSet("images", flag.ExitOnError) 20 | help := "Images prints the paths of all Docker images used by a reflow program." 21 | c.Parse(flags, args, help, "images path") 22 | 23 | if flags.NArg() == 0 { 24 | flags.Usage() 25 | } 26 | programPath := flags.Arg(0) 27 | sess := syntax.NewSession(nil) 28 | m, err := sess.Open(programPath) 29 | c.must(err) 30 | 31 | programFlags, err := m.Flags(sess, sess.Values) 32 | c.must(err) 33 | 34 | programFlags.Usage = func() { 35 | fmt.Fprintf(os.Stderr, "usage of %s:\n", programPath) 36 | programFlags.PrintDefaults() 37 | c.Exit(2) 38 | } 39 | err = programFlags.Parse(flags.Args()[1:]) 40 | if err != nil { 41 | c.Fatalf("error parsing module flags: %v", err) 42 | } 43 | 44 | env := sess.Values.Push() 45 | if err = m.FlagEnv(programFlags, env, types.NewEnv()); err != nil { 46 | fmt.Fprintln(os.Stderr, err) 47 | programFlags.Usage() 48 | } 49 | _, err = m.Make(sess, env) 50 | c.must(err) 51 | 52 | images := sess.Images() 53 | sort.Strings(images) 54 | 55 | for _, image := range images { 56 | fmt.Println(image) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /tool/kill.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | ) 11 | 12 | func (c *Cmd) kill(ctx context.Context, args ...string) { 13 | flags := flag.NewFlagSet("kill", flag.ExitOnError) 14 | help := "Kill terminates and frees allocs." 15 | c.Parse(flags, args, help, "kill allocs...") 16 | if flags.NArg() == 0 { 17 | flags.Usage() 18 | } 19 | cluster := c.CurrentPool(ctx) 20 | for _, arg := range flags.Args() { 21 | n, err := parseName(arg) 22 | if err != nil { 23 | c.Printf("invalid argument '%s': %v", arg, err) 24 | continue 25 | } 26 | if n.Kind != allocName { 27 | c.Errorf("'%s' not an alloc\n", arg) 28 | continue 29 | } 30 | allocUri := allocURI(n) 31 | alloc, err := cluster.Alloc(ctx, allocUri) 32 | if err != nil { 33 | c.Errorf("alloc %s (from arg %s): %v\n", allocUri, arg, err) 34 | continue 35 | } 36 | if err := alloc.Free(ctx); err != nil { 37 | c.Errorf("%s: %s\n", arg, err) 38 | continue 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /tool/limit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | const ( 8 | // The amount of outstanding number of transfers 9 | // between each repository. 10 | defaultTransferLimit = 20 11 | 12 | // The number of concurrent stat operations that can 13 | // be performed against a repository. 14 | statLimit = 200 15 | ) 16 | 17 | // TransferLimit returns the configured transfer limit. 18 | func (c *Cmd) TransferLimit() int { 19 | lim := c.Config.Value("transferlimit") 20 | if lim == nil { 21 | return defaultTransferLimit 22 | } 23 | v, ok := lim.(int) 24 | if !ok { 25 | c.Fatalf("non-integer limit %v", lim) 26 | } 27 | return v 28 | } 29 | -------------------------------------------------------------------------------- /tool/serve.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | 11 | "github.com/grailbio/reflow/reflowlet" 12 | ) 13 | 14 | func (c *Cmd) serveCmd(ctx context.Context, args ...string) { 15 | var ( 16 | flags = flag.NewFlagSet("serve", flag.ExitOnError) 17 | help = `Runs the reflow process in 'reflowlet' which is an agent process. 18 | It exposes a Reflow pool through a REST API. A single Reflowlet can 19 | serve multiple Reflow invocations at any given time. 20 | 21 | In a typical configuration, Reflowlets are automatically launched 22 | through Reflow's ec2cluster mechanism, but they may also be launched 23 | manually if one wishes to outsource cluster management. 24 | 25 | Flag -config defines a configuration filename from which the Reflowlet 26 | restores its configuration. When run in an automatic cluster configuration, 27 | the configuration is typically sealed, containing both configuration information 28 | as well as credentials to access various services. 29 | ` 30 | ) 31 | server := reflowlet.NewServer(c.Version, c.Config) 32 | server.AddFlags(flags) 33 | c.Parse(flags, args, help, "serve [-ec2cluster]") 34 | if flags.NArg() > 0 { 35 | flags.Usage() 36 | } 37 | go reflowlet.IgnoreSigpipe() 38 | // Shutdown the server if the context is done. 39 | go func() { 40 | <-ctx.Done() 41 | server.Shutdown() 42 | }() 43 | c.Fatal(server.ListenAndServe()) 44 | } 45 | -------------------------------------------------------------------------------- /tool/shell.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "bufio" 9 | "context" 10 | "flag" 11 | "io" 12 | "os" 13 | ) 14 | 15 | func (c *Cmd) shell(ctx context.Context, args ...string) { 16 | flags := flag.NewFlagSet("shell", flag.ExitOnError) 17 | help := `Run a shell (/bin/bash) inside the container of a running exec. 18 | The local standard input, output and error streams are attached. 19 | The user may exit the terminal by typing 'exit'/'quit'. 20 | 21 | Note that exec URIs are of the form host:port/alloc/exec.` 22 | // TODO(pgopal) - Put the terminal in raw mode. 23 | c.Parse(flags, args, help, "shell exec") 24 | if flags.NArg() != 1 { 25 | flags.Usage() 26 | } 27 | arg := flags.Arg(0) 28 | n, err := parseName(arg) 29 | if err != nil { 30 | c.Printf("invalid argument '%s', must be one of the following examples:\n%s", arg, objNameExamples) 31 | c.Fatal(err) 32 | } 33 | if n.Kind != execName { 34 | c.Fatalf("%s: not an exec URI", arg) 35 | } 36 | 37 | cluster := c.CurrentPool(ctx) 38 | alloc, err := cluster.Alloc(ctx, allocURI(n)) 39 | if err != nil { 40 | c.Fatalf("alloc %s: %s", allocURI(n), err) 41 | } 42 | e, err := alloc.Get(ctx, n.ID) 43 | if err != nil { 44 | c.Fatalf("%s: %s", n.ID, err) 45 | } 46 | sr, sw := io.Pipe() 47 | go func() { 48 | s := bufio.NewScanner(os.Stdin) 49 | for s.Scan() { 50 | _, err := sw.Write([]byte(s.Text() + "\n")) 51 | if err != nil { 52 | c.Fatalf("%s: %s", n.ID, err) 53 | } 54 | } 55 | sw.Close() 56 | if s.Err() != nil { 57 | c.Fatalf("%s: %s", n.ID, s.Err()) 58 | } 59 | }() 60 | 61 | rwc, err := e.Shell(ctx) 62 | if err != nil { 63 | c.Fatalf("%s: %s", n.ID, err) 64 | } 65 | go func() { 66 | io.Copy(rwc, sr) 67 | }() 68 | _, err = io.Copy(c.Stdout, rwc) 69 | if err != nil { 70 | c.Fatalf("%s: %s", n.ID, err) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /tool/testdata/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "program": "test.rf", 3 | "runs_file": "samples.csv" 4 | } 5 | -------------------------------------------------------------------------------- /tool/testdata/samples.csv: -------------------------------------------------------------------------------- 1 | id,s,i,f,d,b,fl 2 | 1,a,1,s3://grail-marius/1.file,s3://grail-marius/1.dir/,true,1.1 3 | 2,b,2,s3://grail-marius/2.file,s3://grail-marius/2.dir/,false,2.2 4 | -------------------------------------------------------------------------------- /tool/testdata/test.rf: -------------------------------------------------------------------------------- 1 | param ( 2 | s string 3 | i int 4 | f file 5 | d dir 6 | b bool 7 | fl float 8 | 9 | opt string = "ok" 10 | optfile = file("s3://grail-marius") 11 | ) 12 | 13 | @requires(cpu := 1) 14 | val Main = {s, i, f, d, b, fl} 15 | -------------------------------------------------------------------------------- /tool/upgrade.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "io/ioutil" 11 | ) 12 | 13 | func (c *Cmd) upgrade(ctx context.Context, args ...string) { 14 | flags := flag.NewFlagSet("upgrade", flag.ExitOnError) 15 | help := `Upgrade Reflow's configuration and underlying services.` 16 | c.Parse(flags, args, help, "upgrade") 17 | if flags.NArg() != 0 { 18 | flags.Usage() 19 | } 20 | c.must(c.Config.Setup()) 21 | b, err := c.Config.Marshal(false) 22 | c.must(err) 23 | c.must(ioutil.WriteFile(c.ConfigFile, b, 0666)) 24 | } 25 | -------------------------------------------------------------------------------- /tool/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tool 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | "runtime" 12 | ) 13 | 14 | func (c *Cmd) versionCmd(ctx context.Context, args ...string) { 15 | var ( 16 | flags = flag.NewFlagSet("offers", flag.ExitOnError) 17 | help = "Version displays this binary's version (datestamp) and git hash from which it was built." 18 | ) 19 | c.Parse(flags, args, help, "version") 20 | if len(args) != 0 { 21 | flags.Usage() 22 | } 23 | c.Println(c.version()) 24 | } 25 | 26 | func (c *Cmd) version() string { 27 | if c.Version == "" { 28 | c.Version = "broken" 29 | } 30 | if c.Variant != "" { 31 | return fmt.Sprintf("%s (%s, %s)", c.Version, c.Variant, runtime.Version()) 32 | } 33 | return fmt.Sprintf("%s (%s)", c.Version, runtime.Version()) 34 | } 35 | -------------------------------------------------------------------------------- /trace/kind_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Kind"; DO NOT EDIT. 2 | 3 | package trace 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[Run-0] 12 | _ = x[Exec-1] 13 | _ = x[Cache-2] 14 | _ = x[Transfer-3] 15 | _ = x[AllocReq-4] 16 | _ = x[AllocLifespan-5] 17 | } 18 | 19 | const _Kind_name = "RunExecCacheTransferAllocReqAllocLifespan" 20 | 21 | var _Kind_index = [...]uint8{0, 3, 7, 12, 20, 28, 41} 22 | 23 | func (i Kind) String() string { 24 | if i < 0 || i >= Kind(len(_Kind_index)-1) { 25 | return "Kind(" + strconv.FormatInt(int64(i), 10) + ")" 26 | } 27 | return _Kind_name[_Kind_index[i]:_Kind_index[i+1]] 28 | } 29 | -------------------------------------------------------------------------------- /trace/xraytrace/xray_test.go: -------------------------------------------------------------------------------- 1 | package xraytrace 2 | 3 | import ( 4 | "github.com/grailbio/reflow/trace" 5 | 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/grailbio/infra" 10 | ) 11 | 12 | func TestXrayTracerInfra(t *testing.T) { 13 | var schema = infra.Schema{ 14 | "tracer": new(trace.Tracer), 15 | } 16 | config, err := schema.Make(infra.Keys{ 17 | "tracer": "xray", 18 | }) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | var tracer trace.Tracer 23 | config.Must(&tracer) 24 | _, ok := tracer.(*Tracer) 25 | if !ok { 26 | t.Fatalf("%v is not an xraytrace", reflect.TypeOf(t)) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /types/constlevel_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=ConstLevel"; DO NOT EDIT 2 | 3 | package types 4 | 5 | import "fmt" 6 | 7 | const _ConstLevel_name = "NotConstCanConstConst" 8 | 9 | var _ConstLevel_index = [...]uint8{0, 8, 16, 21} 10 | 11 | func (i ConstLevel) String() string { 12 | i -= -1 13 | if i < 0 || i >= ConstLevel(len(_ConstLevel_index)-1) { 14 | return fmt.Sprintf("ConstLevel(%d)", i+-1) 15 | } 16 | return _ConstLevel_name[_ConstLevel_index[i]:_ConstLevel_index[i+1]] 17 | } 18 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package reflow 2 | 3 | import ( 4 | "fmt" 5 | "github.com/grailbio/base/digest" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | // Rundir returns the "runs" directory where reflow run artifacts can be found. 12 | // Rundir returns an error if a it cannot be found (or created). 13 | func Rundir() (string, error) { 14 | var rundir string 15 | if home, ok := os.LookupEnv("HOME"); ok { 16 | rundir = filepath.Join(home, ".reflow", "runs") 17 | if err := os.MkdirAll(rundir, 0777); err != nil { 18 | return "", err 19 | } 20 | } else { 21 | var err error 22 | rundir, err = ioutil.TempDir("", "prefix") 23 | if err != nil { 24 | return "", fmt.Errorf("failed to create temporary directory: %v", err) 25 | } 26 | } 27 | return rundir, nil 28 | } 29 | 30 | // Runbase returns a base path representing a run with the given id under `Rundir`. 31 | // Runbase should be used to create run-specific artifacts with appropriate file suffix added to the returned base path. 32 | func Runbase(id digest.Digest) (string, error) { 33 | dir, err := Rundir() 34 | if err != nil { 35 | return "", err 36 | } 37 | return filepath.Join(dir, id.Hex()), nil 38 | } 39 | -------------------------------------------------------------------------------- /values/equal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package values 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/grailbio/reflow" 11 | "github.com/grailbio/reflow/types" 12 | ) 13 | 14 | func TestEqual(t *testing.T) { 15 | for _, c := range []struct { 16 | typ *types.T 17 | // val1, val2 are either a T or a func() T to accommodate values that cannot be 18 | // constructed in a single expression. 19 | val1, val2 interface{} 20 | want bool 21 | }{ 22 | { 23 | types.File, 24 | reflow.File{ 25 | ID: reflow.Digester.FromString("contents 1"), 26 | Size: 54321, 27 | }, 28 | reflow.File{ 29 | ID: reflow.Digester.FromString("contents 2"), 30 | Size: 54321, 31 | }, 32 | false, 33 | }, 34 | { 35 | types.File, 36 | reflow.File{ 37 | ID: reflow.Digester.FromString("same contents"), 38 | Size: 54321, 39 | Assertions: reflow.AssertionsFromEntry(reflow.AssertionKey{Subject: "subject", Namespace: "namespace"}, map[string]string{"object": "value"}), 40 | }, 41 | reflow.File{ 42 | ID: reflow.Digester.FromString("same contents"), 43 | Size: 54321, 44 | Assertions: reflow.AssertionsFromEntry(reflow.AssertionKey{Subject: "subject2", Namespace: "namespace2"}, map[string]string{"object2": "different value"}), 45 | }, 46 | true, 47 | }, 48 | } { 49 | var v, w T 50 | if f, ok := c.val1.(func() T); ok { 51 | v = f() 52 | } else { 53 | v = c.val1 54 | } 55 | if f, ok := c.val2.(func() T); ok { 56 | w = f() 57 | } else { 58 | w = c.val2 59 | } 60 | if got, want := Equal(v, w), c.want; got != want { 61 | t.Errorf("got %v, want %v", got, want) 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /values/less_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package values 6 | 7 | import ( 8 | "math/rand" 9 | "testing" 10 | 11 | "github.com/grailbio/reflow" 12 | "github.com/grailbio/reflow/types" 13 | ) 14 | 15 | func makeDir(pathsAndFiles ...interface{}) Dir { 16 | var mDir MutableDir 17 | if len(pathsAndFiles)%2 != 0 { 18 | panic(pathsAndFiles) 19 | } 20 | for i := 0; i < len(pathsAndFiles); i += 2 { 21 | path := pathsAndFiles[i].(string) 22 | file := pathsAndFiles[i+1].(reflow.File) 23 | mDir.Set(path, file) 24 | } 25 | return mDir.Dir() 26 | } 27 | 28 | func TestLess(t *testing.T) { 29 | var ( 30 | f0 = reflow.File{Source: "a"} 31 | f1 = reflow.File{Source: "b"} 32 | f2, f3 reflow.File 33 | r = rand.New(rand.NewSource(0)) 34 | ) 35 | for !f2.ID.Less(f3.ID) { 36 | f2.ID = reflow.Digester.Rand(r) 37 | f3.ID = reflow.Digester.Rand(r) 38 | } 39 | less := []struct{ left, right T }{ 40 | {NewInt(0), NewInt(1)}, 41 | {"", "abc"}, 42 | {NewFloat(0.1), NewFloat(100.3)}, 43 | {f0, f1}, 44 | {f1, f2}, 45 | {f2, f3}, 46 | {makeDir("x", f0), makeDir("x", f1)}, 47 | {makeDir("x", f0, "y", f2), makeDir("x", f1, "y", f3)}, 48 | {makeDir("x", f3), makeDir("x", f3, "y", f0)}, 49 | {List{}, List{f0, f1}}, 50 | {List{f0, f1, f3}, List{f0, f2, f3}}, 51 | {MakeMap(types.String, "1", f0), MakeMap(types.String, "1", f1)}, 52 | {MakeMap(types.String, "1", f0), MakeMap(types.String, "1", f0, "2", f0)}, 53 | {Tuple{"a", NewInt(1), "c"}, Tuple{"a", NewInt(2), "c"}}, 54 | {Struct{"x": NewInt(3), "a": "b"}, Struct{"x": NewInt(3), "a": "c"}}, 55 | {Module{"X": NewInt(3), "A": "b"}, Module{"X": NewInt(3), "A": "c"}}, 56 | } 57 | for _, l := range less { 58 | if !Less(l.left, l.right) { 59 | t.Errorf("expected %v < %v", l.left, l.right) 60 | } else if Less(l.right, l.left) { 61 | t.Errorf("assymetric less! %v < %v", l.right, l.left) 62 | } 63 | if Less(l.left, l.left) { 64 | t.Errorf("value %v is equal, but reported as less", l.left) 65 | } 66 | if Less(l.right, l.right) { 67 | t.Errorf("value %v is equal, but reported as less", l.right) 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /values/map_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package values 6 | 7 | import ( 8 | "math/rand" 9 | "testing" 10 | 11 | "github.com/grailbio/reflow" 12 | "github.com/grailbio/reflow/types" 13 | ) 14 | 15 | func TestMap(t *testing.T) { 16 | m := MakeMap(types.String, "1", NewInt(1), "2", NewInt(2), "3", NewInt(3)) 17 | if got, want := m.Lookup(Digest("1", types.String), "1"), NewInt(1); !Equal(got, want) { 18 | t.Errorf("got %v, want %v", got, want) 19 | } 20 | if got, want := m.Len(), 3; got != want { 21 | t.Errorf("got %v, want %v", got, want) 22 | } 23 | m.Insert(Digest("1", types.String), "1", NewInt(123)) 24 | if got, want := m.Lookup(Digest("1", types.String), "1"), NewInt(123); !Equal(got, want) { 25 | t.Errorf("got %v, want %v", got, want) 26 | } 27 | if got, want := m.Len(), 3; got != want { 28 | t.Errorf("got %v, want %v", got, want) 29 | } 30 | } 31 | 32 | func TestMapEmpty(t *testing.T) { 33 | var m Map 34 | var want T 35 | if got := m.Lookup(Digest("2", types.String), "2"); !Equal(got, want) { 36 | t.Errorf("got %v, want %v", got, want) 37 | } 38 | } 39 | 40 | func TestMapDigestCollision(t *testing.T) { 41 | const N = 100 42 | 43 | r := rand.New(rand.NewSource(0)) 44 | d := reflow.Digester.Rand(r) 45 | 46 | m := new(Map) 47 | expect := make(map[int64]int64) 48 | for i := 0; i < N; i++ { 49 | k, v := r.Int63(), r.Int63() 50 | expect[k] = v 51 | m.Insert(d, NewInt(k), NewInt(v)) 52 | } 53 | for k, v := range expect { 54 | if got, want := m.Lookup(d, NewInt(k)), NewInt(v); !Equal(got, want) { 55 | t.Errorf("got %v, want %v", got, want) 56 | } 57 | } 58 | 59 | // Pick some keys to override and test these too. 60 | var i int 61 | for k := range expect { 62 | if i > 100 { 63 | break 64 | } 65 | i++ 66 | v := r.Int63() 67 | expect[k] = v 68 | m.Insert(d, NewInt(k), NewInt(v)) 69 | } 70 | 71 | for k, v := range expect { 72 | if got, want := m.Lookup(d, NewInt(k)), NewInt(v); !Equal(got, want) { 73 | t.Errorf("got %v, want %v", got, want) 74 | } 75 | } 76 | 77 | if got, want := m.Len(), N; got != want { 78 | t.Errorf("got %v, want %v", got, want) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /values/pretty_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package values 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/grailbio/reflow/types" 11 | ) 12 | 13 | func makeMap(entries map[string]string) *Map { 14 | m := new(Map) 15 | for k, v := range entries { 16 | m.Insert(Digest(k, types.String), k, v) 17 | } 18 | return m 19 | } 20 | 21 | func TestPretty(t *testing.T) { 22 | for _, c := range []struct { 23 | v T 24 | t *types.T 25 | p string 26 | }{ 27 | {List{"hello", "world"}, types.List(types.String), `["hello", "world"]`}, 28 | { 29 | Struct{"a": NewInt(123), "b": Tuple{"ok", NewInt(321)}}, 30 | types.Struct( 31 | &types.Field{Name: "a", T: types.Int}, 32 | &types.Field{Name: "b", T: types.Tuple(&types.Field{T: types.String}, &types.Field{T: types.Int})}), 33 | `{a: 123, b: ("ok", 321)}`, 34 | }, 35 | { 36 | makeMap(map[string]string{"a": "b"}), 37 | types.Map(types.String, types.String), 38 | `["a": "b"]`, 39 | }, 40 | } { 41 | if got, want := Sprint(c.v, c.t), c.p; got != want { 42 | t.Errorf("got %s, want %s", got, want) 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /wg/wg_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package wg 6 | 7 | import "testing" 8 | 9 | const N = 16 10 | 11 | func testInterlocked(t *testing.T, w1, w2 *WaitGroup) { 12 | w1.Add(N) 13 | w2.Add(N) 14 | done := make(chan bool) 15 | for i := 0; i < N; i++ { 16 | go func(i int) { 17 | w1.Done() 18 | <-w2.C() 19 | done <- true 20 | }(i) 21 | } 22 | <-w1.C() 23 | for i := 0; i < N; i++ { 24 | select { 25 | case <-done: 26 | t.Fatal("WaitGroup released too soon") 27 | default: 28 | } 29 | w2.Done() 30 | } 31 | for i := 0; i < N; i++ { 32 | <-done 33 | } 34 | } 35 | 36 | func TestWaitGroup(t *testing.T) { 37 | var w1, w2 WaitGroup 38 | testInterlocked(t, &w1, &w2) 39 | } 40 | --------------------------------------------------------------------------------