├── docs ├── bootstrap.graffle │ ├── data.plist │ ├── image1.png │ └── preview.jpeg ├── impl.md └── bootstrap.svg ├── .errcheck-excludes ├── .gitignore ├── cmd ├── ec2boot │ ├── install.bash │ └── ec2boot.go ├── bigoom │ └── main.go ├── diskbench │ ├── doc.go │ └── diskbench.go ├── bigpi │ └── bigpi.go └── multisys │ └── multisys.go ├── go.mod ├── rpc ├── sizereader.go ├── log.go ├── fault_injector.go ├── client_test.go ├── stats.go ├── server_test.go ├── client.go └── server.go ├── regress ├── regress_test.go ├── test1 │ └── test1.go └── teststream │ └── teststream.go ├── internal ├── ioutil │ └── ioutil.go ├── filebuf │ ├── filebuf_test.go │ └── filebuf.go ├── tee │ ├── writer_test.go │ └── writer.go └── authority │ ├── authority_test.go │ └── authority.go ├── .github └── workflows │ └── ci.yml ├── ec2system ├── tmpl_test.go ├── cloudconfig_test.go ├── config.go ├── ec2machine_test.go ├── cloudconfig.go └── internal │ └── monitor │ ├── monitor_test.go │ └── monitor.go ├── driver └── driver.go ├── testsystem ├── testsystem_test.go └── testsystem.go ├── expvar.go ├── .golangci.yml ├── doc.go ├── system.go ├── status.go ├── local.go ├── profile.go ├── machine_test.go ├── LICENSE ├── README.md └── supervisor.go /docs/bootstrap.graffle/data.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/bigmachine/HEAD/docs/bootstrap.graffle/data.plist -------------------------------------------------------------------------------- /docs/bootstrap.graffle/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/bigmachine/HEAD/docs/bootstrap.graffle/image1.png -------------------------------------------------------------------------------- /docs/bootstrap.graffle/preview.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grailbio/bigmachine/HEAD/docs/bootstrap.graffle/preview.jpeg -------------------------------------------------------------------------------- /.errcheck-excludes: -------------------------------------------------------------------------------- 1 | // (*io.PipeWriter).CloseWithError "never overwrites the previous error if it 2 | // exists and always returns nil". 3 | // 4 | // https://golang.org/pkg/io/#PipeWriter.CloseWithError 5 | (*io.PipeWriter).CloseWithError 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | -------------------------------------------------------------------------------- /cmd/ec2boot/install.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | VERSION=ec2boot0.5 7 | 8 | GOOS=linux GOARCH=amd64 go build -o /tmp/$VERSION . 9 | cloudkey ti-apps/admin aws s3 cp --acl public-read /tmp/$VERSION s3://grail-public-bin/linux/amd64/$VERSION 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/grailbio/bigmachine 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.29.24 7 | github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3 8 | github.com/grailbio/base v0.0.9 9 | github.com/grailbio/testutil v0.0.3 10 | github.com/shirou/gopsutil v2.19.9+incompatible 11 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 12 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b 13 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e 14 | golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 15 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 16 | gopkg.in/yaml.v2 v2.2.4 17 | ) 18 | -------------------------------------------------------------------------------- /cmd/ec2boot/ec2boot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Command ec2boot is a minimal bigmachine binary that is intended 6 | // for bootstrapping binaries on EC2. It is used by ec2machine in 7 | // this way. 8 | package main 9 | 10 | import ( 11 | "flag" 12 | 13 | "github.com/grailbio/base/log" 14 | "github.com/grailbio/bigmachine" 15 | "github.com/grailbio/bigmachine/ec2system" 16 | ) 17 | 18 | func main() { 19 | log.AddFlags() 20 | flag.Parse() 21 | bigmachine.Start(ec2system.Instance) 22 | log.Fatal("bigmachine.Start returned") 23 | } 24 | -------------------------------------------------------------------------------- /rpc/sizereader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package rpc 6 | 7 | import "io" 8 | 9 | // SizeTrackingReader keeps track of the number of bytes read 10 | // through the underlying reader. 11 | type sizeTrackingReader struct { 12 | io.Reader 13 | n int 14 | } 15 | 16 | // Read implements io.Reader. 17 | func (s *sizeTrackingReader) Read(p []byte) (n int, err error) { 18 | n, err = s.Reader.Read(p) 19 | s.n += n 20 | return 21 | } 22 | 23 | // Len returns the total number of bytes read from the 24 | // underlying reader. 25 | func (s *sizeTrackingReader) Len() int { return s.n } 26 | -------------------------------------------------------------------------------- /regress/regress_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package regress 6 | 7 | import ( 8 | "os/exec" 9 | "path/filepath" 10 | "testing" 11 | ) 12 | 13 | func TestRegress(t *testing.T) { 14 | t.Parallel() 15 | tests, err := filepath.Glob("test*/*") 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | for _, test := range tests { 20 | test := test 21 | t.Run(test, func(t *testing.T) { 22 | t.Parallel() 23 | cmd := exec.Command("go", "run", test) 24 | if out, err := cmd.CombinedOutput(); err != nil { 25 | t.Errorf("%s\n%s", err, string(out)) 26 | } 27 | }) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /rpc/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package rpc 6 | 7 | import ( 8 | "github.com/grailbio/base/log" 9 | "golang.org/x/time/rate" 10 | ) 11 | 12 | // RateLimitingOutputter is a log.Outputter that enforces a rate 13 | // limit on outputted messages. Messages that are logged beyond 14 | // the allowed rate are dropped. 15 | type rateLimitingOutputter struct { 16 | *rate.Limiter 17 | log.Outputter 18 | } 19 | 20 | // Output implements log.Outputter. 21 | func (r *rateLimitingOutputter) Output(calldepth int, level log.Level, s string) error { 22 | if !r.Limiter.Allow() { 23 | return nil 24 | } 25 | return r.Outputter.Output(calldepth+1, level, s) 26 | } 27 | -------------------------------------------------------------------------------- /internal/ioutil/ioutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package ioutil contains utilities for performing I/O in bigmachine. 6 | package ioutil 7 | 8 | import "io" 9 | 10 | type closingReader struct { 11 | rc io.ReadCloser 12 | closed bool 13 | } 14 | 15 | // NewClosingReader returns a reader that closes the provided 16 | // ReadCloser once it is read through EOF. 17 | func NewClosingReader(rc io.ReadCloser) io.Reader { 18 | return &closingReader{rc, false} 19 | } 20 | 21 | func (c *closingReader) Read(p []byte) (n int, err error) { 22 | if c.closed { 23 | return 0, io.EOF 24 | } 25 | n, err = c.rc.Read(p) 26 | if err == io.EOF { 27 | c.rc.Close() 28 | c.closed = true 29 | } 30 | return 31 | } 32 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | go: [1.12, 1.13, 1.14, 1.15] 15 | os: [ubuntu-latest, macos-latest] 16 | name: Build & Test 17 | runs-on: ${{ matrix.os }} 18 | steps: 19 | - name: Set up Go ${{ matrix.go }} 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: ${{ matrix.go }} 23 | - name: Check out 24 | uses: actions/checkout@v2 25 | - name: Test 26 | run: go test ./... 27 | golangci: 28 | name: Lint 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Check out 32 | uses: actions/checkout@v2 33 | - name: golangci-lint 34 | uses: golangci/golangci-lint-action@v2 35 | with: 36 | version: v1.29 37 | only-new-issues: true 38 | -------------------------------------------------------------------------------- /ec2system/tmpl_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2system 6 | 7 | import "testing" 8 | 9 | func TestTmpl(t *testing.T) { 10 | got := tmpl(` 11 | x, y, {{.z}} 12 | blah 13 | bloop 14 | `, args{"z": 1}) 15 | want := `x, y, 1 16 | blah 17 | bloop` 18 | if got != want { 19 | t.Errorf("got %q, want %q", got, want) 20 | } 21 | } 22 | 23 | func TestBadTmpl(t *testing.T) { 24 | v := recoverPanic(func() { 25 | tmpl(` 26 | x, y 27 | blah`, nil) 28 | }) 29 | if v == nil { 30 | t.Fatal("expected panic") 31 | } 32 | if got, want := v.(string), `nonspace prefix in "\t\t\tblah"`; got != want { 33 | t.Errorf("got %v, want %v", got, want) 34 | } 35 | } 36 | 37 | func recoverPanic(f func()) (v interface{}) { 38 | defer func() { 39 | v = recover() 40 | }() 41 | f() 42 | return 43 | } 44 | -------------------------------------------------------------------------------- /regress/test1/test1.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "encoding/gob" 10 | 11 | "github.com/grailbio/base/log" 12 | "github.com/grailbio/bigmachine" 13 | ) 14 | 15 | func init() { 16 | gob.Register(service{}) 17 | } 18 | 19 | type service struct{} 20 | 21 | func (service) Strlen(ctx context.Context, arg string, reply *int) error { 22 | *reply = len(arg) 23 | return nil 24 | } 25 | 26 | func main() { 27 | b := bigmachine.Start(bigmachine.Local) 28 | defer b.Shutdown() 29 | ctx := context.Background() 30 | machines, err := b.Start(ctx, 1, bigmachine.Services{ 31 | "Service": service{}, 32 | }) 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | m := machines[0] 37 | <-m.Wait(bigmachine.Running) 38 | const str = "hello world" 39 | var n int 40 | if err := m.Call(ctx, "Service.Strlen", str, &n); err != nil { 41 | log.Fatal(err) 42 | } 43 | if got, want := n, len(str); got != want { 44 | log.Fatalf("got %v, want %v", got, want) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /docs/impl.md: -------------------------------------------------------------------------------- 1 | # Bigmachine: implementation notes 2 | 3 | This document contains a set of notes about the design choices 4 | in the Bigmachine implementation. 5 | It is a work in progress. 6 | 7 | * [Bootstrapping](#bootstrapping) 8 | 9 | ## Bootstrapping 10 | 11 | Bigmachine manages machines started by the user. 12 | The machines are provisioned by a cloud provider (1, 2, 3), 13 | and then must be bootstrapped to run the user's code. 14 | Machines boot with a simple bootstrap process: 15 | its only task is to receive a binary and execute it. 16 | The driver process uploads itself 17 | (or the [fatbin](https://godoc.org/github.com/grailbio/base/fatbin) 18 | binary that corresponds to the target GOOS/GOARCH combination) (4), 19 | which is then run by the bootstrap server (5). 20 | Finally, when the server has started, 21 | the Bigmachine driver maintains keepalives 22 | to the server (6). 23 | 24 | ![the Bigmachine bootstrap process](https://raw.github.com/grailbio/bigmachine/master/docs/bootstrap.svg?sanitize=true) 25 | 26 | When the Bigmachine driver process fails to maintain its keepalive 27 | (for example, because it was shut down, it crashed, or experienced a network partition), 28 | the target machine shuts itself down. 29 | -------------------------------------------------------------------------------- /internal/filebuf/filebuf_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package filebuf 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "math/rand" 13 | "testing" 14 | 15 | "github.com/grailbio/testutil/expect" 16 | ) 17 | 18 | type fakeReadCloser struct { 19 | io.Reader 20 | closed bool 21 | } 22 | 23 | func (r *fakeReadCloser) Close() error { 24 | r.closed = true 25 | return nil 26 | } 27 | 28 | type errorReader struct { 29 | err error 30 | } 31 | 32 | func (r errorReader) Read(p []byte) (int, error) { 33 | return 0, r.err 34 | } 35 | 36 | // TestFileBuf verifies that we can create, read from, and close a FileBuf. 37 | func TestFileBuf(t *testing.T) { 38 | in := make([]byte, 1<<20) 39 | rand.Read(in) 40 | rc := &fakeReadCloser{ 41 | Reader: bytes.NewReader(append([]byte{}, in...)), 42 | } 43 | b, err := New(rc) 44 | expect.NoError(t, err) 45 | out, err := ioutil.ReadAll(b) 46 | expect.NoError(t, err) 47 | expect.EQ(t, out, in) 48 | err = b.Close() 49 | expect.NoError(t, err) 50 | expect.True(t, rc.closed) 51 | } 52 | 53 | // TestFileBufReadError verifies that an error reading from the underlying 54 | // reader is propagated. 55 | func TestFileBufReadError(t *testing.T) { 56 | r := errorReader{fmt.Errorf("test error")} 57 | _, err := New(r) 58 | expect.HasSubstr(t, err.Error(), "test error") 59 | } 60 | -------------------------------------------------------------------------------- /ec2system/cloudconfig_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2system 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | ) 11 | 12 | func TestCloudconfig(t *testing.T) { 13 | var c cloudConfig 14 | c.Flavor = Flatcar 15 | c.CoreOS.Update.RebootStrategy = "off" 16 | c.AppendFile(CloudFile{"/tmp/x", "0644", "root", "a test file"}) 17 | c.AppendUnit(CloudUnit{Name: "reflowlet", Command: "command", Enable: true, Content: "unit content"}) 18 | var d cloudConfig 19 | d.AppendUnit(CloudUnit{Name: "xxx", Command: "xxxcommand", Enable: false, Content: "xxx content"}) 20 | d.AppendFile(CloudFile{"/tmp/myfile", "0644", "root", "another test file"}) 21 | c.Merge(&d) 22 | out, err := c.Marshal() 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | if got, want := out, []byte(`#cloud-config 27 | write_files: 28 | - path: /tmp/x 29 | permissions: "0644" 30 | owner: root 31 | content: a test file 32 | - path: /tmp/myfile 33 | permissions: "0644" 34 | owner: root 35 | content: another test file 36 | coreos: 37 | update: 38 | reboot-strategy: "off" 39 | units: 40 | - name: reflowlet 41 | command: command 42 | enable: true 43 | content: unit content 44 | - name: xxx 45 | command: xxxcommand 46 | content: xxx content 47 | `); !bytes.Equal(got, want) { 48 | t.Errorf("got %s, want %s", got, want) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /rpc/fault_injector.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "io" 5 | "math/rand" 6 | 7 | "github.com/grailbio/base/errors" 8 | "github.com/grailbio/base/log" 9 | ) 10 | 11 | // InjectFailures causes HTTP responses to be randomly terminated. Only for 12 | // unittesting. 13 | var InjectFailures = false 14 | 15 | // rpcFaultInjector is an io.ReadCloser implementation that wraps another 16 | // io.ReadCloser but inject artificial failures. 17 | type rpcFaultInjector struct { 18 | label string // for logging. 19 | in io.ReadCloser 20 | err error 21 | } 22 | 23 | func (r *rpcFaultInjector) error(err error) { 24 | r.in.Close() 25 | r.in = nil 26 | r.err = err 27 | } 28 | 29 | // Read implements io.Reader. 30 | func (r *rpcFaultInjector) Read(buf []byte) (int, error) { 31 | if r.in == nil { 32 | return 0, r.err 33 | } 34 | x := rand.Float32() 35 | if x < 0.005 { 36 | r.error(errors.E(errors.Net, errors.Retriable, r.label+": test-induced message drop")) 37 | log.Error.Printf("faultinjector %s: dropping message", r.label) 38 | return 0, r.err 39 | } 40 | n, err := r.in.Read(buf) 41 | if x < 0.01 { 42 | nn := int(float64(n) * rand.Float64()) 43 | log.Error.Printf("faultinjector %s: truncating message from %d->%d", r.label, n, nn) 44 | n = nn 45 | err = errors.E(errors.Net, errors.Retriable, r.label+": test-induced message truncation") 46 | r.error(err) 47 | } 48 | return n, err 49 | } 50 | 51 | // Close implements io.Closer. 52 | func (r *rpcFaultInjector) Close() error { 53 | if r.in != nil { 54 | return r.in.Close() 55 | } 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /driver/driver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package driver provides a convenient API for bigmachine drivers, 6 | // which includes configuration by flags. Driver exports the 7 | // bigmachine's diagnostic http handlers on the default ServeMux. 8 | // 9 | // func main() { 10 | // flag.Parse() 11 | // b := driver.Start() 12 | // defer b.shutdown() 13 | // // Driver code 14 | // } 15 | package driver 16 | 17 | import ( 18 | "flag" 19 | "net/http" 20 | 21 | "github.com/grailbio/base/log" 22 | "github.com/grailbio/bigmachine" 23 | "github.com/grailbio/bigmachine/ec2system" 24 | ) 25 | 26 | var ( 27 | systemFlag = flag.String("bigm.system", "local", "system on which to run the bigmachine") 28 | instanceType = flag.String("bigm.ec2type", "m3.medium", "instance type with which to launch a bigmachine EC2 cluster") 29 | ondemand = flag.Bool("bigm.ec2ondemand", false, "use ec2 on-demand instances instead of spot") 30 | ) 31 | 32 | // Start configures a bigmachine System based on the program's flags, 33 | // Sand then starts it. ee bigmachine.Start for more details. 34 | func Start() *bigmachine.B { 35 | sys := bigmachine.Local 36 | switch *systemFlag { 37 | default: 38 | log.Fatalf("unrecognized system %s", *systemFlag) 39 | case "ec2": 40 | sys = &ec2system.System{ 41 | InstanceType: *instanceType, 42 | OnDemand: *ondemand, 43 | } 44 | case "local": 45 | } 46 | b := bigmachine.Start(sys) 47 | b.HandleDebug(http.DefaultServeMux) 48 | return b 49 | } 50 | -------------------------------------------------------------------------------- /cmd/bigoom/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Command bigoom causes a bigmachine instance to OOM. The sole purpose 6 | // of this binary is to test bigmachine's OOM handling. 7 | package main 8 | 9 | import ( 10 | "context" 11 | "encoding/gob" 12 | "flag" 13 | "net/http" 14 | 15 | "github.com/grailbio/base/log" 16 | "github.com/grailbio/base/stress/oom" 17 | "github.com/grailbio/bigmachine" 18 | "github.com/grailbio/bigmachine/driver" 19 | ) 20 | 21 | func init() { 22 | gob.Register(oomer{}) 23 | } 24 | 25 | type oomer struct{} 26 | 27 | func (oomer) Try(ctx context.Context, _ struct{}, _ *struct{}) error { 28 | oom.Try() 29 | panic("not reached") 30 | } 31 | 32 | func main() { 33 | log.AddFlags() 34 | flag.Parse() 35 | b := driver.Start() 36 | defer b.Shutdown() 37 | 38 | go func() { 39 | err := http.ListenAndServe(":3333", nil) 40 | log.Printf("http.ListenAndServe: %v", err) 41 | }() 42 | ctx := context.Background() 43 | services := bigmachine.Services{ 44 | "OOM": oomer{}, 45 | } 46 | machines, err := b.Start(ctx, 1, services) 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | log.Print("waiting for machines to come online") 51 | m := machines[0] 52 | <-m.Wait(bigmachine.Running) 53 | log.Printf("machine %s %s", m.Addr, m.State()) 54 | if err = m.Err(); err != nil { 55 | log.Fatal(err) 56 | } 57 | 58 | err = m.RetryCall(ctx, "OOM.Try", struct{}{}, nil) 59 | log.Printf("call error: %v", err) 60 | log.Printf("machine error: %v", m.Err()) 61 | } 62 | -------------------------------------------------------------------------------- /testsystem/testsystem_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package testsystem 6 | 7 | import ( 8 | "context" 9 | "encoding/gob" 10 | "testing" 11 | 12 | "github.com/grailbio/base/errors" 13 | "github.com/grailbio/bigmachine" 14 | ) 15 | 16 | func init() { 17 | gob.Register(&testService{}) 18 | } 19 | 20 | type testService struct { 21 | Index int 22 | } 23 | 24 | func (t *testService) Method(ctx context.Context, arg int, reply *int) error { 25 | *reply = t.Index 26 | return nil 27 | } 28 | 29 | func TestTestSystem(t *testing.T) { 30 | test := New() 31 | b := bigmachine.Start(test) 32 | defer b.Shutdown() 33 | ctx := context.Background() 34 | machines, err := b.Start(ctx, 1, bigmachine.Services{ 35 | "Service": &testService{Index: 1}, 36 | }) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | m := machines[0] 41 | <-m.Wait(bigmachine.Running) 42 | var reply int 43 | if err = m.Call(ctx, "Service.Method", 0, &reply); err != nil { 44 | t.Fatal(err) 45 | } 46 | if got, want := reply, 1; got != want { 47 | t.Errorf("got %v, want %v", got, want) 48 | } 49 | if got, want := test.N(), 1; got != want { 50 | t.Errorf("got %v, want %v", got, want) 51 | } 52 | if !test.Kill(nil) { 53 | t.Fatal("failed to kill random machine") 54 | } 55 | if got, want := test.N(), 0; got != want { 56 | t.Errorf("got %v, want %v", got, want) 57 | } 58 | err = m.Call(ctx, "Service.Method", 0, &reply) 59 | if err == nil || !errors.Is(errors.Net, err) { 60 | t.Errorf("bad error %v", err) 61 | } 62 | m.Wait(bigmachine.Stopped) 63 | } 64 | -------------------------------------------------------------------------------- /internal/tee/writer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package tee 6 | 7 | import ( 8 | "io" 9 | "io/ioutil" 10 | "sync" 11 | "testing" 12 | ) 13 | 14 | func write(t *testing.T, w io.Writer, p string) { 15 | t.Helper() 16 | if _, err := io.WriteString(w, p); err != nil { 17 | t.Fatal(err) 18 | } 19 | } 20 | 21 | func read(t *testing.T, r io.Reader, want string) { 22 | t.Helper() 23 | p := make([]byte, len(want)) 24 | if _, err := io.ReadFull(r, p); err != nil { 25 | t.Fatal(err) 26 | } 27 | if got := string(p); got != want { 28 | t.Errorf("got %v, want %v", got, want) 29 | } 30 | } 31 | 32 | func TestWriter(t *testing.T) { 33 | r, w := io.Pipe() 34 | tee := new(Writer) 35 | cancel := tee.Tee(w) 36 | _ = tee.Tee(ioutil.Discard) 37 | 38 | // This should not block, and should not discard. 39 | write(t, tee, "hello, world") 40 | write(t, tee, "hi there") 41 | 42 | read(t, r, "hello, worldhi there") 43 | 44 | cancel() 45 | write(t, tee, "into the void") 46 | var wg sync.WaitGroup 47 | wg.Add(1) 48 | go func() { 49 | read(t, r, ".") 50 | wg.Done() 51 | }() 52 | write(t, w, ".") 53 | wg.Wait() 54 | } 55 | 56 | func TestWriterDiscard(t *testing.T) { 57 | r, w := io.Pipe() 58 | tee := new(Writer) 59 | cancel := tee.Tee(w) 60 | defer cancel() 61 | write(t, tee, "hello") 62 | buf := make([]byte, bufferSize) 63 | read(t, r, "hel") // make sure that we start reading the buffered write 64 | write(t, tee, string(buf)) 65 | write(t, tee, "hello world") 66 | read(t, r, "lo"+string(buf[:len(buf)-len("hello world")])+"hello world") 67 | } 68 | -------------------------------------------------------------------------------- /regress/teststream/teststream.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "encoding/gob" 11 | "io" 12 | "io/ioutil" 13 | "net/http" 14 | _ "net/http/pprof" 15 | "os" 16 | "runtime/pprof" 17 | "time" 18 | 19 | "github.com/grailbio/base/log" 20 | "github.com/grailbio/bigmachine" 21 | "golang.org/x/net/http2" 22 | ) 23 | 24 | func init() { 25 | gob.Register(service{}) 26 | } 27 | 28 | type service struct{} 29 | 30 | func (service) Empty(ctx context.Context, howlong time.Duration, reply *io.ReadCloser) error { 31 | http2.VerboseLogs = true 32 | go func() { 33 | if err := http.ListenAndServe("localhost:8090", nil); err != nil { 34 | log.Fatal(err) 35 | } 36 | }() 37 | *reply = ioutil.NopCloser(bytes.NewReader(nil)) 38 | return nil 39 | } 40 | 41 | func main() { 42 | b := bigmachine.Start(bigmachine.Local) 43 | defer b.Shutdown() 44 | ctx := context.Background() 45 | machines, err := b.Start(ctx, 1, bigmachine.Services{ 46 | "Service": service{}, 47 | }) 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | m := machines[0] 52 | <-m.Wait(bigmachine.Running) 53 | var rc io.ReadCloser 54 | if err = m.Call(ctx, "Service.Empty", time.Second, &rc); err != nil { 55 | log.Fatal(err) 56 | } 57 | go func() { 58 | time.Sleep(3 * time.Second) 59 | // Best effort attempt to write out goroutines for diagnosing. 60 | _ = pprof.Lookup("goroutine").WriteTo(os.Stderr, 1) 61 | log.Fatal("should be dead by now") 62 | }() 63 | if _, err := io.Copy(ioutil.Discard, rc); err != nil { 64 | log.Fatal(err) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /expvar.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigmachine 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "sync" 11 | "time" 12 | 13 | "github.com/grailbio/base/log" 14 | "golang.org/x/sync/errgroup" 15 | ) 16 | 17 | type machineVars struct{ *B } 18 | 19 | // String returns a JSON-formatted string representing the exported 20 | // variables of all underlying machines. 21 | // 22 | // TODO(marius): aggregate values too? 23 | func (v machineVars) String() string { 24 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 25 | defer cancel() 26 | g, ctx := errgroup.WithContext(ctx) 27 | var ( 28 | mu sync.Mutex 29 | vars = make(map[string]Expvars) 30 | ) 31 | for _, m := range v.B.Machines() { 32 | // Only propagate stats for machines we own, otherwise we can 33 | // create stats loops. 34 | if !m.Owned() { 35 | continue 36 | } 37 | 38 | m := m 39 | g.Go(func() error { 40 | var mvars Expvars 41 | if err := m.Call(ctx, "Supervisor.Expvars", struct{}{}, &mvars); err != nil { 42 | log.Error.Printf("failed to retrieve variables for %s: %v", m.Addr, err) 43 | return nil 44 | } 45 | mu.Lock() 46 | vars[m.Addr] = mvars 47 | mu.Unlock() 48 | return nil 49 | }) 50 | } 51 | if err := g.Wait(); err != nil { 52 | b, err2 := json.Marshal(err.Error()) 53 | if err2 != nil { 54 | log.Error.Printf("machineVars marshal: %v", err2) 55 | return `"error"` 56 | } 57 | return string(b) 58 | } 59 | b, err := json.Marshal(vars) 60 | if err != nil { 61 | log.Error.Printf("machineVars marshal: %v", err) 62 | return `"error"` 63 | } 64 | return string(b) 65 | } 66 | -------------------------------------------------------------------------------- /internal/filebuf/filebuf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package filebuf 6 | 7 | import ( 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | 12 | "github.com/grailbio/base/errors" 13 | ) 14 | 15 | // FileBuf is a file-backed buffer that buffers the entire contents of reader. 16 | // This is useful for fully buffering a network read with low memory overhead. 17 | type FileBuf struct { 18 | // file is the temporary file that backs this buffer. 19 | file *os.File 20 | } 21 | 22 | // New creates a new file-backed buffer containing the entire contents of r. If 23 | // there is an error reading from r, an error is returned. If r is an io.Closer, 24 | // r is closed once it is fully read. 25 | func New(r io.Reader) (b *FileBuf, err error) { 26 | if rc, ok := r.(io.Closer); ok { 27 | defer rc.Close() 28 | } 29 | file, err := ioutil.TempFile("", "bigmachine-filebuf-") 30 | if err != nil { 31 | return nil, errors.E("error opening temp file for filebuf", err) 32 | } 33 | defer func() { 34 | if err != nil { 35 | // We created the temporary file but had some other downstream 36 | // error, so we clean up the file now instead of in Close. 37 | os.Remove(file.Name()) 38 | } 39 | }() 40 | _, err = io.Copy(file, r) 41 | if err != nil { 42 | return nil, errors.E("error reading into filebuf", err) 43 | } 44 | _, err = file.Seek(0, io.SeekStart) 45 | if err != nil { 46 | return nil, errors.E("error seeking in filebuf", err) 47 | } 48 | return &FileBuf{file: file}, nil 49 | } 50 | 51 | // Read implements (io.Reader).Read. 52 | func (b *FileBuf) Read(p []byte) (int, error) { 53 | return b.file.Read(p) 54 | } 55 | 56 | // Close implements (io.Closer).Close. 57 | func (b *FileBuf) Close() error { 58 | if b.file == nil { 59 | return nil 60 | } 61 | defer os.Remove(b.file.Name()) 62 | err := b.file.Close() 63 | b.file = nil 64 | return err 65 | } 66 | -------------------------------------------------------------------------------- /cmd/diskbench/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | diskbench runs disk benchmarks on a bigmachine machine. A single bigmachine 3 | machine is started on which benchmarks are run, and results are reported to 4 | stdout. Configuration is read from $HOME/grail/profile and command-line flags. 5 | 6 | Here are current results: 7 | 8 | Configuration: 9 | ami = "ami-1ee65166" // Ubuntu 16.04 10 | instance = "m5.2xlarge" 11 | dataspace = 256 12 | 13 | Results: 14 | $TMPDIR is /mnt/data 15 | === 16 | === benchmark run 1 of 3 17 | === 18 | = writing and reading with dd 19 | 1024+0 records in 20 | 1024+0 records out 21 | 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 1.41673 s, 758 MB/s 22 | 1024+0 records in 23 | 1024+0 records out 24 | 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 0.168252 s, 6.4 GB/s 25 | = running hdparm -Tt 26 | 27 | /dev/md0: 28 | Timing cached reads: 16824 MB in 2.00 seconds = 8432.85 MB/sec 29 | Timing buffered disk reads: 2176 MB in 3.00 seconds = 724.15 MB/sec 30 | === 31 | === benchmark run 2 of 3 32 | === 33 | = writing and reading with dd 34 | 1024+0 records in 35 | 1024+0 records out 36 | 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 1.68509 s, 637 MB/s 37 | 1024+0 records in 38 | 1024+0 records out 39 | 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 0.16413 s, 6.5 GB/s 40 | = running hdparm -Tt 41 | 42 | /dev/md0: 43 | Timing cached reads: 18140 MB in 1.99 seconds = 9093.72 MB/sec 44 | Timing buffered disk reads: 2172 MB in 3.01 seconds = 722.48 MB/sec 45 | === 46 | === benchmark run 3 of 3 47 | === 48 | = writing and reading with dd 49 | 1024+0 records in 50 | 1024+0 records out 51 | 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 1.69384 s, 634 MB/s 52 | 1024+0 records in 53 | 1024+0 records out 54 | 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 0.16817 s, 6.4 GB/s 55 | = running hdparm -Tt 56 | 57 | /dev/md0: 58 | Timing cached reads: 16820 MB in 2.00 seconds = 8430.45 MB/sec 59 | Timing buffered disk reads: 2178 MB in 3.00 seconds = 725.04 MB/sec 60 | */ 61 | package main 62 | -------------------------------------------------------------------------------- /internal/tee/writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package tee implements utilities for I/O multiplexing. 6 | package tee 7 | 8 | import ( 9 | "bytes" 10 | "io" 11 | "sync" 12 | ) 13 | 14 | const bufferSize = 512 << 10 15 | 16 | // Write is an asynchronous write multiplexer. Write maintains a 17 | // small internal buffer but may discard writes for writers that 18 | // cannot keep up; thus it is meant for log output or similar. 19 | type Writer sync.Map 20 | 21 | // Tee forwards future writes from this writer to wr; forwarding 22 | // stops after the return cancelation function is invoked. A small 23 | // buffer is maintained for each writer, but writes are dropped if 24 | // the writer wr cannot keep up with the write volume. The writer is 25 | // not forwarded any more writes if returns an error. 26 | func (w *Writer) Tee(wr io.Writer) (cancel func()) { 27 | c := make(chan *bytes.Buffer, 1) 28 | (*sync.Map)(w).Store(c, nil) 29 | done := make(chan struct{}) 30 | var once sync.Once 31 | cancel = func() { 32 | once.Do(func() { 33 | (*sync.Map)(w).Delete(c) 34 | close(done) 35 | }) 36 | } 37 | go func() { 38 | for { 39 | select { 40 | case <-done: 41 | return 42 | case buf := <-c: 43 | if _, err := io.Copy(wr, buf); err != nil { 44 | cancel() 45 | return 46 | } 47 | } 48 | } 49 | }() 50 | return cancel 51 | } 52 | 53 | // Write writes p to each writer that is managed by this multiplexer. 54 | // Write is asynchronous, and always returns len(p), nil. 55 | func (w *Writer) Write(p []byte) (n int, err error) { 56 | (*sync.Map)(w).Range(func(k, v interface{}) bool { 57 | c := k.(chan *bytes.Buffer) 58 | var buf *bytes.Buffer 59 | select { 60 | case buf = <-c: 61 | default: 62 | buf = new(bytes.Buffer) 63 | } 64 | // Discard extra bytes in the case of buffer overflow. 65 | if n := buf.Len() + len(p) - bufferSize; n > 0 { 66 | buf.Next(n) 67 | } 68 | buf.Write(p) 69 | c <- buf 70 | return true 71 | }) 72 | return len(p), nil 73 | } 74 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | concurrency: 4 3 | timeout: 1m 4 | 5 | # exit code when at least one issue was found 6 | issues-exit-code: 1 7 | 8 | # include test files 9 | tests: true 10 | 11 | build-tags: 12 | 13 | # which dirs to skip: issues from them won't be reported; 14 | skip-dirs: 15 | 16 | # enables skipping of default directories: 17 | # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ 18 | skip-dirs-use-default: true 19 | 20 | # which files to skip: they will be analyzed, but issues from them won't be 21 | # reported. 22 | skip-files: 23 | 24 | # disallow multiple parallel golangci-lint instances 25 | allow-parallel-runners: false 26 | 27 | output: 28 | # colored-line-number|line-number|json|tab|checkstyle|code-climate 29 | format: colored-line-number 30 | 31 | # print lines of code with issue 32 | print-issued-lines: true 33 | 34 | # print linter name in the end of issue text 35 | print-linter-name: true 36 | 37 | # make issues output unique by line 38 | uniq-by-line: true 39 | 40 | linters-settings: 41 | errcheck: 42 | # do not report about not checking errors in type assertions: `a := 43 | # b.(MyStruct)` 44 | check-type-assertions: false 45 | 46 | # do not report about assignment of errors to blank identifier: `num, _ := 47 | # strconv.Atoi(numStr)` 48 | check-blank: false 49 | 50 | # path to a file containing a list of functions to exclude from checking 51 | # see https://github.com/kisielk/errcheck#excluding-functions for details 52 | exclude: .errcheck-excludes 53 | 54 | govet: 55 | # report about shadowed variables 56 | check-shadowing: true 57 | 58 | # settings per analyzer 59 | settings: 60 | # run `go tool vet help` to see all analyzers 61 | printf: 62 | # run `go tool vet help printf` to see available settings for `printf` 63 | # analyzer 64 | funcs: 65 | - (github.com/grailbio/base/log).Fatal 66 | - (github.com/grailbio/base/log).Output 67 | - (github.com/grailbio/base/log).Outputf 68 | - (github.com/grailbio/base/log).Panic 69 | - (github.com/grailbio/base/log).Panicf 70 | - (github.com/grailbio/base/log).Print 71 | - (github.com/grailbio/base/log).Printf 72 | 73 | unused: 74 | # do not report unused exported identifiers 75 | check-exported: false 76 | 77 | misspell: 78 | locale: US 79 | 80 | linters: 81 | disable-all: true 82 | fast: false 83 | enable: 84 | - deadcode 85 | - goimports 86 | - gosimple 87 | - govet 88 | - errcheck 89 | - ineffassign 90 | - misspell 91 | - staticcheck 92 | - structcheck 93 | - typecheck 94 | - unused 95 | - varcheck 96 | -------------------------------------------------------------------------------- /rpc/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package rpc 6 | 7 | import ( 8 | "context" 9 | "io" 10 | "net/http" 11 | "net/http/httptest" 12 | "testing" 13 | 14 | "github.com/grailbio/base/errors" 15 | ) 16 | 17 | var fatalErr = errors.E(errors.Fatal) 18 | 19 | func TestNetError(t *testing.T) { 20 | url, client := newTestClient(t) 21 | e := errors.E(errors.Net, "some network error") 22 | err := client.Call(context.Background(), url, "Test.ErrorError", e, nil) 23 | if err == nil { 24 | t.Error("expected error") 25 | } else if !errors.Is(errors.Remote, err) { 26 | t.Errorf("error %v is not a remote error", err) 27 | } else if !errors.Match(e, errors.Recover(err).Err) { 28 | t.Errorf("error %v does not match expected error %v", err, e) 29 | } 30 | } 31 | 32 | // TestClientError verifies that client errors (4XXs) are handled appropriately. 33 | func TestClientError(t *testing.T) { 34 | url, client := newTestClient(t) 35 | // Cause a (client) error by using an int instead of a string argument. This is a 36 | // bad request that is not a temporary condition (i.e. should not be retried). 37 | var notAString int 38 | err := client.Call(context.Background(), url, "Test.Echo", notAString, nil) 39 | if err == nil { 40 | t.Fatal("expected error") 41 | } 42 | if !errors.Match(fatalErr, err) { 43 | t.Errorf("error %v is not fatal", err) 44 | } 45 | } 46 | 47 | // TestEncodeError verifies that errors encoding arguments are handled 48 | // appropriately. 49 | func TestEncodeError(t *testing.T) { 50 | url, client := newTestClient(t) 51 | type teapot struct { 52 | // nolint: structcheck,unused 53 | unexported int 54 | } 55 | // teapot will cause an encoding error because it has no exported fields. 56 | err := client.Call(context.Background(), url, "Test.Echo", teapot{}, nil) 57 | if err == nil { 58 | t.Fatal("expected error") 59 | } 60 | if !errors.Match(fatalErr, err) { 61 | t.Errorf("error %v is not fatal", err) 62 | } 63 | } 64 | 65 | // TestReaderFuncArgError verifies that a (func() (io.Reader, error)) arg 66 | // passed to Call that returns an error causes the appropriate error handling. 67 | func TestReaderFuncArgError(t *testing.T) { 68 | url, client := newTestClient(t) 69 | makeReader := func() (io.Reader, error) { 70 | return nil, errors.E(errors.Fatal, "test error") 71 | } 72 | err := client.Call(context.Background(), url, "Test.Echo", makeReader, nil) 73 | if err == nil { 74 | t.Fatal("expected error") 75 | } 76 | if !errors.Match(fatalErr, err) { 77 | t.Errorf("error %v is not fatal", err) 78 | } 79 | } 80 | 81 | // newTestClient returns the address of a server running the TestService and a 82 | // client for calling that server. 83 | func newTestClient(t *testing.T) (string, *Client) { 84 | t.Helper() 85 | srv := NewServer() 86 | if err := srv.Register("Test", new(TestService)); err != nil { 87 | t.Fatal(err) 88 | } 89 | httpsrv := httptest.NewServer(srv) 90 | client, err := NewClient(func() *http.Client { return httpsrv.Client() }, testPrefix) 91 | if err != nil { 92 | t.Fatal(err) 93 | } 94 | return httpsrv.URL, client 95 | } 96 | -------------------------------------------------------------------------------- /rpc/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package rpc 6 | 7 | import ( 8 | "expvar" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | var serverstats, clientstats rpcstats 14 | 15 | func init() { 16 | expvar.Publish("server", &serverstats) 17 | expvar.Publish("client", &clientstats) 18 | } 19 | 20 | // A treestats represents a tree of expvars. 21 | type treestats struct { 22 | expvar.Map 23 | mu sync.Mutex 24 | } 25 | 26 | // Path returns the treestats with the provided path. 27 | func (t *treestats) Path(names ...string) *treestats { 28 | child := t 29 | for _, name := range names { 30 | child = child.Child(name) 31 | } 32 | return child 33 | } 34 | 35 | // Child returns the treestat's child with the given path, 36 | // creating one if it does not yet exist. 37 | func (t *treestats) Child(name string) *treestats { 38 | child, ok := t.Map.Get(name).(*treestats) 39 | if child != nil && ok { 40 | return child 41 | } 42 | t.mu.Lock() 43 | defer t.mu.Unlock() 44 | child, ok = t.Map.Get(name).(*treestats) 45 | if child != nil && ok { 46 | return child 47 | } 48 | child = new(treestats) 49 | t.Map.Set(name, child) 50 | return child 51 | } 52 | 53 | // Rpcstats maintains simple RPC statistics, aggregated by address 54 | // and method. 55 | type rpcstats struct { 56 | treestats 57 | } 58 | 59 | // Start starts an RPC stat with the provided address and method. It returns a 60 | // function that records the status and latency of the RPC. The caller must run 61 | // the function after the RPC finishes. Args {request,reply}Bytes report the 62 | // sizes of the RPC payloads. They may be -1 if the size is unknown (e.g., when 63 | // the RPC is streaming). Arg err is the result of the RPC. 64 | func (r *rpcstats) Start(addr, method string) (done func(requestBytes, replyBytes int64, err error)) { 65 | r.Path("method", method).Add("count", 1) 66 | if addr != "" { 67 | r.Path("machine", addr, "method", method).Add("count", 1) 68 | } 69 | now := time.Now() 70 | return func(requestBytes, replyBytes int64, err error) { 71 | elapsed := int64(time.Since(now).Nanoseconds()) / 1e6 72 | r.Path("method", method).Add("time", elapsed) 73 | if requestBytes > 0 { 74 | r.Path("method", method).Add("requestbytes", requestBytes) 75 | r.max(requestBytes, "method", method, "maxrequestbytes") 76 | } 77 | if replyBytes > 0 { 78 | r.Path("method", method).Add("replybytes", replyBytes) 79 | r.max(replyBytes, "method", method, "maxreplybytes") 80 | } 81 | if err != nil { 82 | r.Path("method", method).Add("errors", 1) 83 | } 84 | r.max(elapsed, "method", method, "maxtime") 85 | 86 | if addr != "" { 87 | r.Path("machine", addr, "method", method).Add("time", elapsed) 88 | r.max(elapsed, "machine", addr, "method", method, "maxtime") 89 | } 90 | } 91 | } 92 | 93 | func (r *rpcstats) max(val int64, path ...string) { 94 | path, name := path[:len(path)-1], path[len(path)-1] 95 | r.Path(path...).Add(name, 0) 96 | if iv, ok := r.Path(path...).Get(name).(*expvar.Int); ok { 97 | if val > iv.Value() { 98 | iv.Set(val) 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /ec2system/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2system 6 | 7 | import ( 8 | "strings" 9 | 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/grailbio/base/config" 12 | "github.com/grailbio/base/errors" 13 | ) 14 | 15 | // Defaults for the ec2boot binary. These are used when the "binary" value is empty. 16 | // For backwards compatibility (old configs), any binary with the prefix 17 | // defaultEc2BootPrefix is rewritten to the current version. 18 | const ( 19 | defaultEc2BootPrefix = "https://grail-public-bin.s3-us-west-2.amazonaws.com/linux/amd64/ec2boot" 20 | defaultEc2BootVersion = "0.5" 21 | defaultEc2Boot = defaultEc2BootPrefix + defaultEc2BootVersion 22 | ) 23 | 24 | func init() { 25 | config.Register("bigmachine/ec2system", func(constr *config.Constructor[*System]) { 26 | var system System 27 | 28 | // TODO(marius): maybe defer defaults to system impl? 29 | constr.BoolVar(&system.OnDemand, "ondemand", false, "use only on-demand instances") 30 | constr.BoolVar(&system.SpotOnly, "spot-only", false, "use only spot instances; overrides ondemand") 31 | constr.StringVar(&system.InstanceType, "instance", "m3.medium", "instance type to allocate") 32 | // Flatcar-stable-2512.2.1-hvm 33 | constr.StringVar(&system.AMI, "ami", "ami-0bb54692374ac10a7", "AMI to bootstrap") 34 | 35 | flavor := constr.String("flavor", "flatcar", "one of {flatcar, ubuntu}") 36 | constr.StringVar(&system.InstanceProfile, "instance-profile", "", 37 | "the instance profile with which to launch new instances") 38 | constr.StringVar(&system.SecurityGroup, "security-group", "", 39 | "the security group with which new instances are launched") 40 | constr.StringVar(&system.DefaultRegion, "default-region", "us-west-2", "default AWS region to use when one is not explicitly set via an aws.Config") 41 | diskspace := constr.Int("diskspace", 200, "the amount of (root) disk space to allocate") 42 | dataspace := constr.Int("dataspace", 0, "the amount of scratch/data space to allocate") 43 | constr.StringVar(&system.Binary, "binary", 44 | "", 45 | "the bootstrap bigmachine binary with which machines are launched") 46 | sshkeys := constr.String("sshkey", "", "comma-separated list of ssh keys to be installed") 47 | constr.InstanceVar(&system.Eventer, "eventer", "", "the event logger used to log bigmachine events") 48 | constr.StringVar(&system.Username, "username", "", "user name for tagging purposes") 49 | var sess *session.Session 50 | constr.InstanceVar(&sess, "aws", "aws", "AWS configuration for all EC2 calls") 51 | constr.Doc = "bigmachine/ec2system configures the default instances settings used for bigmachine's ec2 backend" 52 | constr.New = func() (*System, error) { 53 | switch *flavor { 54 | case "flatcar": 55 | system.Flavor = Flatcar 56 | case "ubuntu": 57 | system.Flavor = Ubuntu 58 | default: 59 | return nil, errors.E(errors.Invalid, "flavor must be one of {flatcar, ubuntu}: ", *flavor) 60 | } 61 | system.Diskspace = uint(*diskspace) 62 | system.Dataspace = uint(*dataspace) 63 | system.SshKeys = strings.Split(*sshkeys, ",") 64 | system.AWSConfig = sess.Config 65 | return &system, nil 66 | } 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /internal/authority/authority_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package authority_test 6 | 7 | import ( 8 | "crypto/rsa" 9 | "crypto/x509" 10 | "io/ioutil" 11 | "net" 12 | "os" 13 | "reflect" 14 | "testing" 15 | "time" 16 | 17 | "github.com/grailbio/bigmachine/internal/authority" 18 | ) 19 | 20 | func TestCA(t *testing.T) { 21 | name, cleanup := tempFile(t) 22 | defer cleanup() 23 | ca, err := authority.New(name) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | 28 | now := time.Now() 29 | ips := []net.IP{net.IPv4(1, 2, 3, 4)} 30 | dnses := []string{"test.grail.com"} 31 | certBytes, priv, err := ca.Issue("test", 10*time.Minute, ips, dnses) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | verify(t, ca, now, certBytes, priv, ips, dnses) 36 | 37 | // Make sure that when we restore the CA, it's still a valid cert. 38 | ca, err = authority.New(name) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | verify(t, ca, now, certBytes, priv, ips, dnses) 43 | } 44 | 45 | func TestCAEphemeral(t *testing.T) { 46 | ca, err := authority.New("") 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | now := time.Now() 51 | ips := []net.IP{net.IPv4(1, 2, 3, 4)} 52 | dnses := []string{"test2.grail.com"} 53 | certBytes, priv, err := ca.Issue("test", 10*time.Minute, ips, dnses) 54 | if err != nil { 55 | t.Fatal(err) 56 | } 57 | verify(t, ca, now, certBytes, priv, ips, dnses) 58 | } 59 | 60 | func verify(t *testing.T, ca *authority.T, now time.Time, certBytes []byte, priv *rsa.PrivateKey, ips []net.IP, dnses []string) { 61 | t.Helper() 62 | cert, err := x509.ParseCertificate(certBytes) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | opts := x509.VerifyOptions{} 67 | opts.Roots = x509.NewCertPool() 68 | opts.Roots.AddCert(ca.Cert()) 69 | if _, err := cert.Verify(opts); err != nil { 70 | t.Fatal(err) 71 | } 72 | if err := priv.Validate(); err != nil { 73 | t.Fatal(err) 74 | } 75 | if got, want := priv.Public(), cert.PublicKey; !reflect.DeepEqual(got, want) { 76 | t.Errorf("got %v, want %v", got, want) 77 | } 78 | if got, want := cert.Subject.CommonName, "test"; got != want { 79 | t.Errorf("got %q, want %q", got, want) 80 | } 81 | if got, want := cert.NotBefore, now.Add(-authority.DriftMargin); want.Before(got) { 82 | t.Errorf("wanted %s <= %s", got, want) 83 | } 84 | if got, want := cert.NotAfter.Sub(cert.NotBefore), 10*time.Minute+authority.DriftMargin; got != want { 85 | t.Errorf("got %s, want %s", got, want) 86 | } 87 | if cert.IsCA { 88 | t.Error("cert is CA") 89 | } 90 | if got, want := cert.IPAddresses, ips; !ipsEqual(got, want) { 91 | t.Errorf("got %v, want %v", got, want) 92 | } 93 | if got, want := cert.DNSNames, dnses; !reflect.DeepEqual(got, want) { 94 | t.Errorf("got %v, want %v", got, want) 95 | } 96 | } 97 | 98 | func tempFile(t *testing.T) (string, func()) { 99 | // Test that the CA generates valid certs. 100 | f, err := ioutil.TempFile("", "") 101 | if err != nil { 102 | t.Fatal(err) 103 | } 104 | name := f.Name() 105 | if err := f.Close(); err != nil { 106 | t.Fatal(err) 107 | } 108 | if err := os.Remove(name); err != nil { 109 | t.Fatal(err) 110 | } 111 | return name, func() { os.Remove(name) } 112 | } 113 | 114 | func ipsEqual(x, y []net.IP) bool { 115 | if len(x) != len(y) { 116 | return false 117 | } 118 | for i := range x { 119 | if !x[i].Equal(y[i]) { 120 | return false 121 | } 122 | } 123 | return true 124 | } 125 | -------------------------------------------------------------------------------- /ec2system/ec2machine_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2system 6 | 7 | import ( 8 | "fmt" 9 | "net" 10 | "net/http" 11 | "path/filepath" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "github.com/grailbio/base/errors" 17 | "github.com/grailbio/bigmachine/internal/authority" 18 | "github.com/grailbio/testutil" 19 | "golang.org/x/net/http2" 20 | ) 21 | 22 | func TestDiskConfig(t *testing.T) { 23 | for _, test := range []struct { 24 | dataspace uint 25 | nslice int 26 | sliceSize int64 27 | }{ 28 | {256, 16, 16}, 29 | {257, 17, 16}, 30 | {1000, 25, 40}, 31 | {5350, 25, 214}, 32 | {5350 + 25, 25, 215}, 33 | {6000, 25, 240}, 34 | } { 35 | sys := System{Dataspace: test.dataspace} 36 | nslice, sliceSize := sys.sliceConfig() 37 | if got, want := nslice, test.nslice; got != want { 38 | t.Errorf("%+v: got %v, want %v", test, got, want) 39 | } 40 | if got, want := sliceSize, test.sliceSize; got != want { 41 | t.Errorf("%+v: got %v, want %v", test, got, want) 42 | } 43 | } 44 | } 45 | 46 | func TestMutualHTTPS(t *testing.T) { 47 | save := useInstanceIDSuffix 48 | useInstanceIDSuffix = false 49 | defer func() { 50 | useInstanceIDSuffix = save 51 | }() 52 | // This is a really nasty way of testing what's going on here, 53 | // but we do want to test this property end-to-end. 54 | mux := new(http.ServeMux) 55 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 56 | fmt.Fprintln(w, "ok") 57 | }) 58 | 59 | port, err := getFreeTCPPort() 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | 64 | temp, cleanup := testutil.TempDir(t, "", "") 65 | defer cleanup() 66 | 67 | sys := new(System) 68 | sys.authority, err = authority.New(filepath.Join(temp, "authority")) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | // Create a second, unrelated authority. Clients from this should not be able 73 | // to communicate with the first. 74 | authority, err := authority.New(filepath.Join(temp, "authority2")) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | 79 | var listenAndServeError errors.Once 80 | go func() { 81 | listenAndServeError.Set(sys.ListenAndServe(fmt.Sprintf("localhost:%d", port), mux)) 82 | }() 83 | time.Sleep(time.Second) 84 | 85 | config, _, err := authority.HTTPSConfig() 86 | if err != nil { 87 | t.Fatal(err) 88 | } 89 | transport := &http.Transport{TLSClientConfig: config} 90 | if err = http2.ConfigureTransport(transport); err != nil { 91 | t.Fatal(err) 92 | } 93 | client := &http.Client{Transport: transport} 94 | _, err = client.Get(fmt.Sprintf("https://localhost:%d/", port)) 95 | if err == nil { 96 | t.Fatal("expected error") 97 | } 98 | // We expect to see a certificate error, but we occasionally see a broken 99 | // pipe, presumably because the server closes the connection while we are 100 | // still writing. This is claimed to be fixed[1] at least in part, but we 101 | // still see the behavior, possibly because of some subtlety in our setup. 102 | // 103 | // [1] https://github.com/golang/go/issues/15709 104 | if !strings.Contains(err.Error(), "remote error: tls: bad certificate") && 105 | !strings.Contains(err.Error(), "broken pipe") { 106 | t.Fatalf("bad error %v", err) 107 | } 108 | if err := listenAndServeError.Err(); err != nil { 109 | t.Fatal(err) 110 | } 111 | } 112 | 113 | func getFreeTCPPort() (int, error) { 114 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 115 | if err != nil { 116 | return 0, err 117 | } 118 | l, err := net.ListenTCP("tcp", addr) 119 | if err != nil { 120 | return 0, err 121 | } 122 | port := l.Addr().(*net.TCPAddr).Port 123 | l.Close() 124 | return port, nil 125 | } 126 | -------------------------------------------------------------------------------- /cmd/diskbench/diskbench.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/gob" 6 | "fmt" 7 | "io" 8 | "os" 9 | "os/exec" 10 | "path" 11 | "strings" 12 | 13 | "github.com/grailbio/base/config" 14 | "github.com/grailbio/base/grail" 15 | "github.com/grailbio/base/log" 16 | "github.com/grailbio/base/must" 17 | "github.com/grailbio/bigmachine" 18 | _ "github.com/grailbio/bigmachine/ec2system" 19 | "github.com/grailbio/bigmachine/rpc" 20 | ) 21 | 22 | func main() { 23 | bigmachine.Init() 24 | grail.Init() 25 | var sys bigmachine.System 26 | must.Nil(config.Instance("bigmachine", &sys)) 27 | b := bigmachine.Start(sys) 28 | defer b.Shutdown() 29 | ctx := context.Background() 30 | machines, err := b.Start(ctx, 1, bigmachine.Services{"Bench": bench{}}) 31 | must.Nil(err, "starting machines") 32 | log.Print("waiting for machine") 33 | m := machines[0] 34 | <-m.Wait(bigmachine.Running) 35 | log.Print("running benchmark") 36 | var rc io.ReadCloser 37 | must.Nil(m.Call(ctx, "Bench.Run", struct{}{}, &rc)) 38 | defer func() { 39 | must.Nil(rc.Close()) 40 | }() 41 | _, err = io.Copy(os.Stdout, rc) 42 | must.Nil(err) 43 | } 44 | 45 | func init() { 46 | gob.Register(bench{}) 47 | } 48 | 49 | type bench struct{} 50 | 51 | func (bench) Run(ctx context.Context, _ struct{}, rc *io.ReadCloser) error { 52 | r, w := io.Pipe() 53 | *rc = rpc.Flush(r) 54 | go func() { 55 | if err := run(w); err != nil { 56 | if closeErr := w.CloseWithError(err); closeErr != nil { 57 | log.Error.Printf("closing pipe writer: %v", closeErr) 58 | } 59 | return 60 | } 61 | if err := w.Close(); err != nil { 62 | log.Printf("closing pipe writer: %v", err) 63 | } 64 | }() 65 | return nil 66 | } 67 | 68 | func run(w io.Writer) error { 69 | var ( 70 | msg string 71 | tmpDir = os.Getenv("TMPDIR") 72 | ) 73 | if tmpDir == "" { 74 | msg = "$TMPDIR empty; assuming /tmp" 75 | tmpDir = "/tmp" 76 | } else { 77 | msg = fmt.Sprintf("$TMPDIR is %s\n", tmpDir) 78 | } 79 | if _, err := io.WriteString(w, msg); err != nil { 80 | return fmt.Errorf("writing $TMPDIR value: %v", err) 81 | } 82 | dev, err := resolveDev(tmpDir) 83 | if err != nil { 84 | return fmt.Errorf("resolving device of %s: %v", tmpDir, err) 85 | } 86 | const N = 3 87 | for i := 0; i < N; i++ { 88 | status := fmt.Sprintf("===\n=== benchmark run %d of %d\n===\n", i+1, N) 89 | if _, err = io.WriteString(w, status); err != nil { 90 | return fmt.Errorf("writing status: %v", err) 91 | } 92 | if err := dd(w, tmpDir); err != nil { 93 | return fmt.Errorf("running dd: %v", err) 94 | } 95 | if err := hdparm(w, dev); err != nil { 96 | return fmt.Errorf("running hdparm: %v", err) 97 | } 98 | } 99 | return nil 100 | } 101 | 102 | func resolveDev(p string) (string, error) { 103 | cmd := exec.Command("findmnt", "-n", "-o", "SOURCE", "--target", p) 104 | out, err := cmd.Output() 105 | if err != nil { 106 | return "", fmt.Errorf("running findmnt to resolve %s: %v", p, err) 107 | } 108 | return strings.TrimRight(string(out), "\n"), nil 109 | } 110 | 111 | func dd(w io.Writer, tmpDir string) error { 112 | if _, err := io.WriteString(w, "= writing and reading with dd\n"); err != nil { 113 | return fmt.Errorf("writing status: %v", err) 114 | } 115 | p := path.Join(tmpDir, "bench.tmp") 116 | if err := runCmd(w, "dd", 117 | "if=/dev/zero", 118 | fmt.Sprintf("of=%s", p), 119 | "conv=fdatasync", 120 | "bs=1M", 121 | "count=1024", 122 | ); err != nil { 123 | return fmt.Errorf("writing with dd: %v", err) 124 | } 125 | if err := runCmd(w, "dd", 126 | fmt.Sprintf("if=%s", p), 127 | "of=/dev/null", 128 | "bs=1M", 129 | "count=1024", 130 | ); err != nil { 131 | return fmt.Errorf("reading with dd: %v", err) 132 | } 133 | return nil 134 | } 135 | 136 | func hdparm(w io.Writer, dev string) error { 137 | if _, err := io.WriteString(w, "= running hdparm -Tt\n"); err != nil { 138 | return fmt.Errorf("writing status: %v", err) 139 | } 140 | return runCmd(w, "hdparm", "-Tt", dev) 141 | } 142 | 143 | func runCmd(w io.Writer, name string, arg ...string) error { 144 | cmd := exec.Command(name, arg...) 145 | cmd.Stdout = w 146 | cmd.Stderr = w 147 | return cmd.Run() 148 | } 149 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package bigmachine implements a vertically integrated stack for 7 | distributed computing in Go. Go programs written with bigmachine are 8 | transparently distributed across a number of machines as 9 | instantiated by the backend used. (Currently supported: EC2, local 10 | machines, unit tests.) Bigmachine clusters comprise a driver node 11 | and a number of bigmachine nodes (called "machines"). The driver 12 | node can create new machines and communicate with them; machines can 13 | call each other. 14 | 15 | Computing model 16 | 17 | On startup, a bigmachine program calls driver.Start. Driver.Start 18 | configures a bigmachine instance based on a set of standard flags 19 | and then starts it. (Users desiring a lower-level API can use 20 | bigmachine.Start directly.) 21 | 22 | import ( 23 | "github.com/grailbio/bigmachine" 24 | "github.com/grailbio/bigmachine/driver" 25 | ... 26 | ) 27 | 28 | func main() { 29 | flag.Parse() 30 | // Additional configuration and setup. 31 | b, shutdown := driver.Start() 32 | defer shutdown() 33 | 34 | // Driver code... 35 | } 36 | 37 | When the program is run, driver.Start returns immediately: the program 38 | can then interact with the returned bigmachine B to create new 39 | machines, define services on those machines, and invoke methods on 40 | those services. Bigmachine bootstraps machines by running the same 41 | binary, but in these runs, driver.Start never returns; instead it 42 | launches a server to handle calls from the driver program and other 43 | machines. 44 | 45 | A machine is started by (*B).Start. Machines must be configured with 46 | at least one service: 47 | 48 | m, err := b.Start(ctx, bigmachine.Services{ 49 | "MyService": &MyService{Param1: value1, ...}, 50 | }) 51 | 52 | Users may then invoke methods on the services provided by the 53 | returned machine. A services's methods can be invoked so long as 54 | they are of the form: 55 | 56 | Func(ctx context.Context, arg argType, reply *replyType) error 57 | 58 | See package github.com/grailbio/bigmachine/rpc for more details. 59 | 60 | Methods are named by the sevice and method name, separated by a dot 61 | ('.'), e.g.: "MyService.MyMethod": 62 | 63 | if err := m.Call(ctx, "MyService.MyMethod", arg, &reply); err != nil { 64 | log.Print(err) 65 | } else { 66 | // Examine reply 67 | } 68 | 69 | Since service instances must be serialized so that they can be transmitted 70 | to the remote machine, and because we do not know the service types 71 | a priori, any type that can appear as a service must be registered with 72 | gob. This is usually done in an init function in the package that declares 73 | type: 74 | 75 | type MyService struct { ... } 76 | 77 | func init() { 78 | // MyService has method receivers 79 | gob.Register(new(MyService)) 80 | } 81 | 82 | 83 | Vertical computing 84 | 85 | A bigmachine program attempts to appear and act like a single program: 86 | 87 | - Each machine's standard output and error are copied to the driver; 88 | - bigmachine provides aggregating profile handlers at /debug/bigmachine/pprof 89 | so that aggregate profiles may be taken over the full cluster; 90 | - command line flags are propagated from the driver to the machine, 91 | so that a binary run can be configured in the usual way. 92 | 93 | The driver program maintains keepalives to all of its machines. Once 94 | this is no longer maintained (e.g., because the driver finished, or 95 | crashed, or lost connectivity), the machines become idle and shut 96 | down. 97 | 98 | 99 | Services 100 | 101 | A service is any Go value that implements methods of the form given 102 | above. Services are instantiated by the user and registered with 103 | bigmachine. When a service is registered, bigmachine will also 104 | invoke an initialization method on the service if it exists. 105 | Per-machine initialization can be performed by this method.The form 106 | of the method is: 107 | 108 | Init(*Service) error 109 | 110 | If a non-nil error is returned, the machine is considered failed. 111 | */ 112 | package bigmachine 113 | -------------------------------------------------------------------------------- /ec2system/cloudconfig.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package ec2system 6 | 7 | import ( 8 | "fmt" 9 | "path" 10 | 11 | yaml "gopkg.in/yaml.v2" 12 | ) 13 | 14 | // CloudFile is a component of the cloudConfig configuration as accepted by 15 | // cloud-init. It represents a file that will be written to the filesystem. 16 | type CloudFile struct { 17 | Path string `yaml:"path,omitempty"` 18 | Permissions string `yaml:"permissions,omitempty"` 19 | Owner string `yaml:"owner,omitempty"` 20 | Content string `yaml:"content,omitempty"` 21 | } 22 | 23 | // CloudUnit is a component of the cloudConfig configuration as accepted by 24 | // cloud-init. It represents a systemd unit. 25 | type CloudUnit struct { 26 | Name string `yaml:"name,omitempty"` 27 | Command string `yaml:"command,omitempty"` 28 | Enable bool `yaml:"enable,omitempty"` 29 | Content string `yaml:"content,omitempty"` 30 | 31 | // Sync determines whether the command should be run synchronously. 32 | Sync bool `yaml:"-"` 33 | } 34 | 35 | // cloudConfig represents a cloud configuration as accepted by 36 | // cloud-init. CloudConfigs can be incrementally defined and then 37 | // rendered by its Marshal method. 38 | type cloudConfig struct { 39 | // Flavor indicates the flavor of cloud-config; it determines 40 | // how Systemd units are processed before serialization. 41 | Flavor Flavor `yaml:"-"` 42 | 43 | WriteFiles []CloudFile `yaml:"write_files,omitempty"` 44 | CoreOS struct { 45 | Update struct { 46 | RebootStrategy string `yaml:"reboot-strategy,omitempty"` 47 | } `yaml:"update,omitempty"` 48 | Units []CloudUnit `yaml:"units,omitempty"` 49 | } `yaml:"coreos,omitempty"` 50 | SshAuthorizedKeys []string `yaml:"ssh_authorized_keys,omitempty"` 51 | 52 | // RunCmd stores a list of cloud-init run commands. 53 | RunCmd []string `yaml:"runcmd,omitempty"` 54 | // Mounts stores a list of cloud-init mounts. 55 | Mounts [][]string `yaml:"mounts,omitempty"` 56 | 57 | units []CloudUnit 58 | } 59 | 60 | // Merge merges cloudConfig d into c. List entries from c are 61 | // appended to d, and key-values are overwritten. 62 | func (c *cloudConfig) Merge(d *cloudConfig) { 63 | c.WriteFiles = append(c.WriteFiles, d.WriteFiles...) 64 | if s := d.CoreOS.Update.RebootStrategy; s != "" { 65 | c.CoreOS.Update.RebootStrategy = s 66 | } 67 | c.units = append(c.units, d.units...) 68 | c.SshAuthorizedKeys = append(c.SshAuthorizedKeys, d.SshAuthorizedKeys...) 69 | } 70 | 71 | // AppendFile appends the file f to the cloudConfig c. 72 | func (c *cloudConfig) AppendFile(f CloudFile) { 73 | c.WriteFiles = append(c.WriteFiles, f) 74 | } 75 | 76 | // AppendUnit appends the systemd unit u to the cloudConfig c. 77 | func (c *cloudConfig) AppendUnit(u CloudUnit) { 78 | c.units = append(c.units, u) 79 | } 80 | 81 | // AppendRunCmd appends a run command to the cloud config. 82 | // Note that run commands are only respected in the Ubuntu 83 | // flavor. 84 | func (c *cloudConfig) AppendRunCmd(cmd string) { 85 | c.RunCmd = append(c.RunCmd, cmd) 86 | } 87 | 88 | // AppendMount appends a mount spec. Note that mounts are 89 | // only respected in the Ubuntu flavor. 90 | func (c *cloudConfig) AppendMount(mount []string) { 91 | c.Mounts = append(c.Mounts, mount) 92 | } 93 | 94 | // Marshal renders the cloudConfig into YAML, with the prerequisite 95 | // cloud-config header. 96 | func (c *cloudConfig) Marshal() ([]byte, error) { 97 | copy := *c 98 | if c.Flavor == Flatcar { 99 | copy.CoreOS.Units = c.units 100 | } else { 101 | if len(c.units) > 0 { 102 | copy.RunCmd = append(copy.RunCmd, "systemctl daemon-reload") 103 | } 104 | for _, u := range c.units { 105 | if u.Content != "" { 106 | copy.AppendFile(CloudFile{ 107 | Path: path.Join("/etc/systemd/system", u.Name), 108 | Permissions: "0644", 109 | Content: u.Content, 110 | }) 111 | } 112 | if u.Sync { 113 | copy.RunCmd = append(copy.RunCmd, fmt.Sprintf("systemctl %s %s", u.Command, u.Name)) 114 | } else { 115 | copy.RunCmd = append(copy.RunCmd, fmt.Sprintf("systemctl --no-block %s %s", u.Command, u.Name)) 116 | } 117 | } 118 | } 119 | 120 | b, err := yaml.Marshal(copy) 121 | if err != nil { 122 | return nil, err 123 | } 124 | return append([]byte("#cloud-config\n"), b...), nil 125 | } 126 | -------------------------------------------------------------------------------- /system.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigmachine 6 | 7 | import ( 8 | "context" 9 | "encoding/gob" 10 | "io" 11 | "net/http" 12 | "os" 13 | "sync" 14 | "time" 15 | 16 | "github.com/grailbio/base/must" 17 | ) 18 | 19 | // A System implements a set of methods to set up a bigmachine and 20 | // start new machines. Systems are also responsible for providing an 21 | // HTTP client that can be used to communicate between machines 22 | // and drivers. 23 | type System interface { 24 | // Name is the name of this system. It is used to multiplex multiple system 25 | // implementations, and thus should be unique among systems. 26 | Name() string 27 | // Init initializes this system for use by a bigmachine.B. For convenience, 28 | // it is called by bigmachine.Start, so implementations must be idempotent 29 | // and return nil in subsequent calls if the first call returned nil. 30 | Init() error 31 | // Main is called to start a machine. The system is expected to take over 32 | // the process; the bigmachine fails if main returns (and if it does, it 33 | // should always return with an error). 34 | Main() error 35 | // Event logs an event of typ with (key, value) fields given in fieldPairs 36 | // as k0, v0, k1, v1, ...kn, vn. For example: 37 | // 38 | // s.Event("bigmachine:machineStart", "addr", "https://m0") 39 | // 40 | // These semi-structured events are used for analytics. 41 | Event(typ string, fieldPairs ...interface{}) 42 | // HTTPClient returns an HTTP client that can be used to communicate 43 | // from drivers to machines as well as between machines. 44 | HTTPClient() *http.Client 45 | // ListenAndServe serves the provided handler on an HTTP server that 46 | // is reachable from other instances in the bigmachine cluster. If addr 47 | // is the empty string, the default cluster address is used. 48 | ListenAndServe(addr string, handle http.Handler) error 49 | // Start launches up to n new machines. The returned machines can be in 50 | // Unstarted state, but should eventually become available. 51 | Start(ctx context.Context, b *B, n int) ([]*Machine, error) 52 | // Exit is called to terminate a machine with the provided exit code. 53 | Exit(int) 54 | // Shutdown is called on graceful driver exit. It's should be used to 55 | // perform system tear down. It is not guaranteed to be called. 56 | Shutdown() 57 | // Maxprocs returns the maximum number of processors per machine, 58 | // as configured. Returns 0 if is a dynamic value. 59 | Maxprocs() int 60 | // KeepaliveConfig returns the various keepalive timeouts that should 61 | // be used to maintain keepalives for machines started by this system. 62 | KeepaliveConfig() (period, timeout, rpcTimeout time.Duration) 63 | // Tail returns a reader that follows the bigmachine process logs. 64 | Tail(ctx context.Context, m *Machine) (io.Reader, error) 65 | // Read returns a reader that reads the contents of the provided filename 66 | // on the host. This is done outside of the supervisor to support external 67 | // monitoring of the host. 68 | Read(ctx context.Context, m *Machine, filename string) (io.Reader, error) 69 | // KeepaliveFailed notifies the system of a failed call to 70 | // Supervisor.Keepalive. This might be an intermittent failure that will be 71 | // retried. The system can use this notification as a hint to otherwise 72 | // probe machine health. 73 | KeepaliveFailed(ctx context.Context, m *Machine) 74 | } 75 | 76 | var ( 77 | systemsMu sync.Mutex 78 | systems = make(map[string]System) 79 | ) 80 | 81 | // RegisterSystem is used by systems implementation to register a 82 | // system implementation. RegisterSystem registers the implementation 83 | // with gob, so that instances can be transmitted over the wire. It 84 | // also registers the provided System instance as a default to use 85 | // for the name to support bigmachine.Init. 86 | func RegisterSystem(name string, system System) { 87 | gob.Register(system) 88 | systemsMu.Lock() 89 | defer systemsMu.Unlock() 90 | must.Nil(systems[name], "system ", name, " already registered") 91 | systems[name] = system 92 | } 93 | 94 | // Init initializes bigmachine. It should be called after flag 95 | // parsing and global setup in bigmachine-based processes. Init is a 96 | // no-op if the binary is not running as a bigmachine worker; if it 97 | // is, Init never returns. 98 | func Init() { 99 | name := os.Getenv("BIGMACHINE_SYSTEM") 100 | if name == "" { 101 | return 102 | } 103 | system, ok := systems[name] 104 | must.True(ok, "system ", name, " not found") 105 | must.Nil(system.Init(), "system initialization error") 106 | must.Never("start returned: ", Start(system)) 107 | } 108 | -------------------------------------------------------------------------------- /ec2system/internal/monitor/monitor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package monitor 6 | 7 | import ( 8 | "context" 9 | "encoding/gob" 10 | "math/rand" 11 | "strconv" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "github.com/aws/aws-sdk-go/aws" 17 | "github.com/aws/aws-sdk-go/aws/request" 18 | "github.com/aws/aws-sdk-go/service/ec2" 19 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 20 | "github.com/grailbio/bigmachine" 21 | "github.com/grailbio/bigmachine/testsystem" 22 | "github.com/grailbio/testutil/assert" 23 | "github.com/grailbio/testutil/expect" 24 | "golang.org/x/time/rate" 25 | ) 26 | 27 | // TestCancellation verifies that machines backed by terminated instances are 28 | // stopped by the monitor. 29 | func TestCancellation(t *testing.T) { 30 | // We (somewhat strangely) attach our monitor to testsystem machines. It 31 | // does not make much sense to tie these machines to (fake) EC2 instances, 32 | // except that it allows us to conveniently test monitoring. 33 | const N = 100 34 | var ( 35 | api = NewAPI() 36 | sys = testsystem.New() 37 | b = bigmachine.Start(sys) 38 | limiter = rate.NewLimiter(rate.Inf, 1) 39 | m = Start(api, limiter) 40 | ) 41 | defer m.Cancel() 42 | machines, err := b.Start(context.Background(), N, bigmachine.Services{ 43 | "Dummy": &dummyService{}, 44 | }) 45 | assert.Nil(t, err) 46 | for _, machine := range machines { 47 | <-machine.Wait(bigmachine.Running) 48 | } 49 | for i, machine := range machines { 50 | var sfx string 51 | switch { 52 | case i%4 == 0: 53 | sfx = "-terminated" 54 | case i%2 == 0: 55 | sfx = "-shutting-down" 56 | } 57 | m.Started(strconv.Itoa(i)+sfx, machine) 58 | } 59 | for _, machine := range machines { 60 | m.KeepaliveFailed(machine) 61 | } 62 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 63 | for i, machine := range machines { 64 | if i%2 == 0 { 65 | // Expect that machines we tagged to be terminated above are 66 | // stopped. 67 | select { 68 | case <-machine.Wait(bigmachine.Stopped): 69 | case <-ctx.Done(): 70 | t.Fatal("took too long") 71 | } 72 | } else { 73 | expect.EQ(t, machine.State(), bigmachine.Running) 74 | } 75 | } 76 | cancel() 77 | } 78 | 79 | // TestRateLimiting verifies that monitoring does not violate the rate limit 80 | // imposed by the passed limiter. 81 | func TestRateLimiting(t *testing.T) { 82 | const ( 83 | Interval = 50 * time.Millisecond 84 | Nmachine = 100 85 | ) 86 | var ( 87 | api = NewAPI() 88 | limiter = rate.NewLimiter(rate.Every(Interval), 1) 89 | m = Start(api, limiter) 90 | ) 91 | defer m.Cancel() 92 | machines := make([]*bigmachine.Machine, Nmachine) 93 | for i := range machines { 94 | machines[i] = &bigmachine.Machine{ 95 | Addr: strconv.Itoa(i), 96 | } 97 | m.Started(strconv.Itoa(i), machines[i]) 98 | } 99 | ctx, cancel := context.WithCancel(context.Background()) 100 | go func() { 101 | for { 102 | time.Sleep(1 * time.Millisecond) 103 | j := rand.Intn(len(machines)) 104 | m.KeepaliveFailed(machines[j]) 105 | select { 106 | case <-ctx.Done(): 107 | return 108 | default: 109 | } 110 | } 111 | }() 112 | time.Sleep(1 * time.Second) 113 | cancel() 114 | expect.LT(t, api.Rate(), float64(1*time.Second)/float64(Interval)) 115 | } 116 | 117 | type fakeEC2API struct { 118 | ec2iface.EC2API 119 | start time.Time 120 | reqCount int 121 | } 122 | 123 | func NewAPI() *fakeEC2API { 124 | return &fakeEC2API{ 125 | start: time.Now(), 126 | } 127 | } 128 | 129 | // Rate returns the number of requests made per second up until the time at 130 | // which it is called. 131 | func (api *fakeEC2API) Rate() float64 { 132 | d := time.Since(api.start) 133 | return float64(api.reqCount) / float64(d.Seconds()) 134 | } 135 | 136 | func (api *fakeEC2API) DescribeInstancesWithContext( 137 | ctx context.Context, 138 | input *ec2.DescribeInstancesInput, 139 | _ ...request.Option, 140 | ) (*ec2.DescribeInstancesOutput, error) { 141 | api.reqCount++ 142 | defer time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond) 143 | instances := make([]*ec2.Instance, len(input.InstanceIds)) 144 | for i := range instances { 145 | code := int64(16) // running 146 | name := ec2.InstanceStateNameRunning 147 | switch { 148 | case strings.HasSuffix(aws.StringValue(input.InstanceIds[i]), "-terminated"): 149 | code = int64(48) // terminated 150 | name = ec2.InstanceStateNameTerminated 151 | case strings.HasSuffix(aws.StringValue(input.InstanceIds[i]), "-shutting-down"): 152 | code = int64(32) // shutting-down 153 | name = ec2.InstanceStateNameShuttingDown 154 | } 155 | instances[i] = &ec2.Instance{ 156 | InstanceId: input.InstanceIds[i], 157 | State: &ec2.InstanceState{ 158 | Code: aws.Int64(code), 159 | Name: aws.String(name), 160 | }, 161 | } 162 | } 163 | return &ec2.DescribeInstancesOutput{ 164 | Reservations: []*ec2.Reservation{ 165 | {Instances: instances}, 166 | }, 167 | }, nil 168 | } 169 | 170 | type dummyService struct{} 171 | 172 | func init() { 173 | gob.Register(&dummyService{}) 174 | } 175 | -------------------------------------------------------------------------------- /status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigmachine 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "io" 11 | "net/http" 12 | "sort" 13 | "strings" 14 | "text/tabwriter" 15 | "text/template" 16 | "time" 17 | 18 | "github.com/grailbio/base/data" 19 | "github.com/grailbio/base/diagnostic/dump" 20 | "golang.org/x/sync/errgroup" 21 | ) 22 | 23 | var startTime = time.Now() 24 | 25 | var statusTemplate = template.Must(template.New("status"). 26 | Funcs(template.FuncMap{ 27 | "roundjoindur": func(ds []time.Duration) string { 28 | strs := make([]string, len(ds)) 29 | for i, d := range ds { 30 | d = d - d%time.Millisecond 31 | strs[i] = d.String() 32 | } 33 | return strings.Join(strs, " ") 34 | }, 35 | "until": time.Until, 36 | "human": func(v interface{}) string { 37 | switch v := v.(type) { 38 | case int: 39 | return data.Size(v).String() 40 | case int64: 41 | return data.Size(v).String() 42 | case uint64: 43 | return data.Size(v).String() 44 | default: 45 | return fmt.Sprintf("(!%T)%v", v, v) 46 | } 47 | }, 48 | "ns": func(v interface{}) string { 49 | switch v := v.(type) { 50 | case int: 51 | return time.Duration(v).String() 52 | case int64: 53 | return time.Duration(v).String() 54 | case uint64: 55 | return time.Duration(v).String() 56 | default: 57 | return fmt.Sprintf("(!%T)%v", v, v) 58 | } 59 | }, 60 | }). 61 | Parse(`{{.machine.Addr}} 62 | {{if .machine.Owned}} keepalive: 63 | next: {{.info.NextKeepalive}} (in {{until .info.NextKeepalive}}) 64 | reply times: {{roundjoindur .info.KeepaliveReplyTimes}} 65 | {{end}} memory: 66 | total: {{human .info.MemInfo.System.Total}} 67 | used: {{human .info.MemInfo.System.Used}} 68 | (percent): {{printf "%.1f%%" .info.MemInfo.System.UsedPercent}} 69 | available: {{human .info.MemInfo.System.Available}} 70 | runtime: {{human .info.MemInfo.Runtime.Sys}} 71 | (alloc): {{human .info.MemInfo.Runtime.Alloc}} 72 | runtime: 73 | uptime: {{.uptime}} 74 | pausetime: {{ns .info.MemInfo.Runtime.PauseTotalNs}} 75 | (last): {{ns .lastpause}} 76 | disk: 77 | total: {{human .info.DiskInfo.Usage.Total}} 78 | available: {{human .info.DiskInfo.Usage.Free}} 79 | used: {{human .info.DiskInfo.Usage.Used}} 80 | (percent): {{printf "%.1f%%" .info.DiskInfo.Usage.UsedPercent}} 81 | load: {{printf "%.1f %.1f %.1f" .info.LoadInfo.Averages.Load1 .info.LoadInfo.Averages.Load5 .info.LoadInfo.Averages.Load15}} 82 | `)) 83 | 84 | func makeStatusDumpFunc(b *B) dump.Func { 85 | return func(ctx context.Context, w io.Writer) error { 86 | return writeStatus(ctx, b, w) 87 | } 88 | } 89 | 90 | func writeStatus(ctx context.Context, b *B, w io.Writer) error { 91 | machines := b.Machines() 92 | sort.Slice(machines, func(i, j int) bool { 93 | return machines[i].Addr < machines[j].Addr 94 | }) 95 | infos := make([]machineInfo, len(machines)) 96 | g, ctx := errgroup.WithContext(ctx) 97 | for i, m := range machines { 98 | if state := m.State(); state != Running { 99 | infos[i].err = fmt.Errorf("machine state %s", state) 100 | continue 101 | } 102 | i, m := i, m 103 | g.Go(func() error { 104 | infos[i] = allInfo(ctx, m) 105 | return nil 106 | }) 107 | } 108 | if err := g.Wait(); err != nil { 109 | return err 110 | } 111 | var tw tabwriter.Writer 112 | tw.Init(w, 4, 4, 1, ' ', 0) 113 | defer tw.Flush() 114 | for i, info := range infos { 115 | m := machines[i] 116 | if info.err != nil { 117 | fmt.Fprintln(&tw, m.Addr, ":", info.err) 118 | continue 119 | } 120 | err := statusTemplate.Execute(&tw, map[string]interface{}{ 121 | "machine": m, 122 | "info": info, 123 | "uptime": time.Since(startTime), 124 | "lastpause": info.MemInfo.Runtime.PauseNs[(info.MemInfo.Runtime.NumGC+255)%256], 125 | }) 126 | if err != nil { 127 | panic(err) 128 | } 129 | } 130 | return nil 131 | } 132 | 133 | // StatusHandler implements an HTTP handler that displays machine 134 | // statuses. 135 | type statusHandler struct{ *B } 136 | 137 | func (s *statusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 138 | w.Header().Set("Content-Type", "text/plain; charset=utf-8") 139 | if err := writeStatus(r.Context(), s.B, w); err != nil { 140 | http.Error(w, fmt.Sprint(err), http.StatusInternalServerError) 141 | } 142 | } 143 | 144 | type machineInfo struct { 145 | err error 146 | MemInfo 147 | DiskInfo 148 | LoadInfo 149 | KeepaliveReplyTimes []time.Duration 150 | NextKeepalive time.Time 151 | } 152 | 153 | func allInfo(ctx context.Context, m *Machine) machineInfo { 154 | ctx, cancel := context.WithTimeout(ctx, 10*time.Second) 155 | defer cancel() 156 | g, ctx := errgroup.WithContext(ctx) 157 | var ( 158 | mem MemInfo 159 | disk DiskInfo 160 | load LoadInfo 161 | ) 162 | g.Go(func() error { 163 | var err error 164 | mem, err = m.MemInfo(ctx, true) 165 | return err 166 | }) 167 | g.Go(func() error { 168 | var err error 169 | disk, err = m.DiskInfo(ctx) 170 | return err 171 | }) 172 | g.Go(func() error { 173 | var err error 174 | load, err = m.LoadInfo(ctx) 175 | return err 176 | }) 177 | err := g.Wait() 178 | return machineInfo{ 179 | err: err, 180 | MemInfo: mem, 181 | DiskInfo: disk, 182 | LoadInfo: load, 183 | KeepaliveReplyTimes: m.KeepaliveReplyTimes(), 184 | NextKeepalive: m.NextKeepalive(), 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /rpc/server_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package rpc 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "crypto" 11 | "crypto/rand" 12 | "io" 13 | "io/ioutil" 14 | "net/http" 15 | "net/http/httptest" 16 | "os" 17 | "strings" 18 | "testing" 19 | 20 | "github.com/grailbio/base/digest" 21 | "github.com/grailbio/base/errors" 22 | ) 23 | 24 | const testPrefix = "/" 25 | 26 | var digester = digest.Digester(crypto.SHA256) 27 | 28 | type TestService struct{} 29 | 30 | func (s *TestService) Echo(ctx context.Context, arg string, reply *string) error { 31 | *reply = arg 32 | return nil 33 | } 34 | 35 | func (s *TestService) Error(ctx context.Context, message string, reply *string) error { 36 | return errors.E(message) 37 | } 38 | 39 | func (s *TestService) ErrorError(ctx context.Context, err *errors.Error, reply *string) error { 40 | return err 41 | } 42 | 43 | func TestServer(t *testing.T) { 44 | srv := NewServer() 45 | if err := srv.Register("Test", new(TestService)); err != nil { 46 | t.Fatal(err) 47 | } 48 | httpsrv := httptest.NewServer(srv) 49 | client, err := NewClient(func() *http.Client { return httpsrv.Client() }, testPrefix) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | ctx := context.Background() 54 | 55 | var reply string 56 | if err = client.Call(ctx, httpsrv.URL, "Test.Echo", "hello world", &reply); err != nil { 57 | t.Fatal(err) 58 | } 59 | if got, want := reply, "hello world"; got != want { 60 | t.Errorf("got %v, want %v", got, want) 61 | } 62 | err = client.Call(ctx, httpsrv.URL, "Test.Error", "the error message", nil) 63 | if err == nil { 64 | t.Fatal("expected error") 65 | } 66 | if !errors.Is(errors.Remote, err) { 67 | t.Errorf("expected remote error") 68 | } 69 | cause := errors.Recover(err).Err 70 | if cause == nil { 71 | t.Fatalf("expected remote error to have a cause") 72 | } 73 | if got, want := cause.Error(), "the error message"; got != want { 74 | t.Errorf("got %v, want %v", got, want) 75 | } 76 | // Just test that nil replies just discard the result. 77 | if err = client.Call(ctx, httpsrv.URL, "Test.Echo", "hello world", nil); err != nil { 78 | t.Error(err) 79 | } 80 | _, err = os.Open("/dev/notexist") 81 | e := errors.E(errors.Precondition, "xyz", err) 82 | err = client.Call(ctx, httpsrv.URL, "Test.ErrorError", e, nil) 83 | if err == nil { 84 | t.Fatal("expected error") 85 | } 86 | if !errors.Is(errors.Remote, err) { 87 | t.Errorf("expected remote error") 88 | } 89 | if !errors.Match(e, errors.Recover(err).Err) { 90 | t.Errorf("error %v does not match expected error %v", err, e) 91 | } 92 | } 93 | 94 | type TestStreamService struct{} 95 | 96 | func (s *TestStreamService) Echo(ctx context.Context, arg io.Reader, reply *io.ReadCloser) error { 97 | r, w := io.Pipe() 98 | go func() { 99 | _, err := io.Copy(w, arg) 100 | w.CloseWithError(err) 101 | }() 102 | *reply = r 103 | return nil 104 | } 105 | 106 | func (s *TestStreamService) StreamWithError(ctx context.Context, errstr string, reply *io.ReadCloser) error { 107 | r, w := io.Pipe() 108 | w.CloseWithError(errors.New(errstr)) 109 | *reply = r 110 | return nil 111 | } 112 | 113 | func (s *TestStreamService) Gimme(ctx context.Context, count int, reply *io.ReadCloser) error { 114 | r, w := io.Pipe() 115 | *reply = r 116 | go func() { 117 | block := make([]byte, 1024) 118 | for count > 0 { 119 | b := block 120 | if count < len(b) { 121 | b = b[:count] 122 | } 123 | n, err := w.Write(b) 124 | if err != nil { 125 | w.CloseWithError(err) 126 | return 127 | } 128 | count -= n 129 | } 130 | w.Close() 131 | }() 132 | return nil 133 | } 134 | 135 | func (s *TestStreamService) Digest(ctx context.Context, arg io.Reader, reply *digest.Digest) error { 136 | w := digester.NewWriter() 137 | if _, err := io.Copy(w, arg); err != nil { 138 | return err 139 | } 140 | *reply = w.Digest() 141 | return nil 142 | } 143 | 144 | func TestStream(t *testing.T) { 145 | srv := NewServer() 146 | if err := srv.Register("Stream", new(TestStreamService)); err != nil { 147 | t.Fatal(err) 148 | } 149 | httpsrv := httptest.NewServer(srv) 150 | client, err := NewClient(func() *http.Client { return httpsrv.Client() }, testPrefix) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | ctx := context.Background() 155 | b := make([]byte, 1024) 156 | if _, err = rand.Read(b); err != nil { 157 | t.Fatal(err) 158 | } 159 | var rc io.ReadCloser 160 | if err = client.Call(ctx, httpsrv.URL, "Stream.Echo", bytes.NewReader(b), &rc); err != nil { 161 | t.Fatal(err) 162 | } 163 | c, err := ioutil.ReadAll(rc) 164 | if err != nil { 165 | t.Fatal(err) 166 | } 167 | if err = rc.Close(); err != nil { 168 | t.Error(err) 169 | } 170 | if !bytes.Equal(b, c) { 171 | t.Errorf("got %v, want %v", c, b) 172 | } 173 | 174 | // Make sure errors are propagated (both ways). 175 | if err = client.Call(ctx, httpsrv.URL, "Stream.StreamWithError", "a series of unfortunate events", &rc); err != nil { 176 | t.Fatal(err) 177 | } 178 | if _, err = io.Copy(ioutil.Discard, rc); err == nil || !strings.Contains(err.Error(), "a series of unfortunate events") { 179 | t.Errorf("bad error %v", err) 180 | } 181 | rc.Close() 182 | 183 | var d digest.Digest 184 | if err = client.Call(ctx, httpsrv.URL, "Stream.Digest", bytes.NewReader(b), &d); err != nil { 185 | t.Fatal(err) 186 | } 187 | if got, want := d, digester.FromBytes(b); got != want { 188 | t.Errorf("got %v, want %v", got, want) 189 | } 190 | 191 | n := 32 << 20 192 | if err = client.Call(ctx, httpsrv.URL, "Stream.Gimme", n, &rc); err != nil { 193 | t.Fatal(err) 194 | } 195 | m, err := io.Copy(ioutil.Discard, rc) 196 | if err != nil { 197 | t.Fatal(err) 198 | } 199 | if err := rc.Close(); err != nil { 200 | t.Error(err) 201 | } 202 | if got, want := m, int64(n); got != want { 203 | t.Errorf("got %v, want %v", got, want) 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /ec2system/internal/monitor/monitor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package monitor 6 | 7 | import ( 8 | "context" 9 | "time" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/service/ec2" 13 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 14 | "github.com/grailbio/base/log" 15 | "github.com/grailbio/bigmachine" 16 | "golang.org/x/time/rate" 17 | ) 18 | 19 | // T is the monitor type. It monitors instances started by the EC2 system. 20 | type T struct { 21 | ec2 ec2iface.EC2API 22 | limiter *rate.Limiter 23 | ch chan event 24 | cancel func() 25 | } 26 | 27 | // machineInstance pairs a machine with its backing instance. 28 | type machineInstance struct { 29 | machine *bigmachine.Machine 30 | instanceID string 31 | } 32 | 33 | type machineInstanceSet map[machineInstance]struct{} 34 | 35 | // event is the type for events processed by the monitoring loop. 36 | type event interface{} 37 | 38 | // eventStarted signals that a machine has been started by the EC2 system. 39 | type eventStarted struct { 40 | instanceID string 41 | machine *bigmachine.Machine 42 | } 43 | 44 | // eventStopped signals that a machine has been stopped. 45 | type eventStopped struct { 46 | machine *bigmachine.Machine 47 | } 48 | 49 | // eventKeepaliveFailed signals that a keepalive call to a machine has failed. 50 | type eventKeepaliveFailed struct { 51 | machine *bigmachine.Machine 52 | } 53 | 54 | // eventBatchDone indicates that processing of the current batch of machines is 55 | // done. 56 | type eventBatchDone struct{} 57 | 58 | // Start starts a monitor. 59 | func Start(ec2 ec2iface.EC2API, limiter *rate.Limiter) *T { 60 | m := &T{ 61 | ec2: ec2, 62 | limiter: limiter, 63 | ch: make(chan event), 64 | } 65 | var ctx context.Context 66 | ctx, m.cancel = context.WithCancel(context.Background()) 67 | go m.loop(ctx) 68 | return m 69 | } 70 | 71 | // Started notifies m of a started machine to be monitored. 72 | func (m *T) Started(instanceID string, machine *bigmachine.Machine) { 73 | m.ch <- eventStarted{instanceID: instanceID, machine: machine} 74 | } 75 | 76 | // KeepaliveFailed notifies m of a failed call to Supervisor.Keepalive. 77 | func (m *T) KeepaliveFailed(machine *bigmachine.Machine) { 78 | m.ch <- eventKeepaliveFailed{machine} 79 | } 80 | 81 | // Cancel cancels monitoring. 82 | func (m *T) Cancel() { 83 | m.cancel() 84 | } 85 | 86 | func (m *T) loop(ctx context.Context) { 87 | var ( 88 | machines = make(map[*bigmachine.Machine]machineInstance) 89 | triggerCh <-chan time.Time 90 | batch = make(machineInstanceSet) 91 | batchInFlight machineInstanceSet 92 | maybeSchedule = func() { 93 | if len(batch) == 0 { 94 | return 95 | } 96 | if triggerCh == nil { 97 | res := m.limiter.Reserve() 98 | if !res.OK() { 99 | panic("limiter too restrictive") 100 | } 101 | triggerCh = time.After(res.Delay()) 102 | } 103 | } 104 | ) 105 | for { 106 | select { 107 | case e := <-m.ch: 108 | switch e := e.(type) { 109 | case eventStarted: 110 | go func() { 111 | select { 112 | case <-e.machine.Wait(bigmachine.Stopped): 113 | case <-ctx.Done(): 114 | return 115 | } 116 | select { 117 | case m.ch <- eventStopped{e.machine}: 118 | case <-ctx.Done(): 119 | return 120 | } 121 | }() 122 | machines[e.machine] = machineInstance{ 123 | machine: e.machine, 124 | instanceID: e.instanceID, 125 | } 126 | case eventStopped: 127 | delete(machines, e.machine) 128 | case eventKeepaliveFailed: 129 | mi, ok := machines[e.machine] 130 | if !ok { 131 | continue 132 | } 133 | if batchInFlight != nil { 134 | if _, ok := batchInFlight[mi]; ok { 135 | // We are already checking the machine, so do nothing 136 | // with this notification. 137 | continue 138 | } 139 | batch[mi] = struct{}{} 140 | continue 141 | } 142 | batch[mi] = struct{}{} 143 | maybeSchedule() 144 | case eventBatchDone: 145 | batchInFlight = nil 146 | maybeSchedule() 147 | } 148 | case <-triggerCh: 149 | triggerCh = nil 150 | batchInFlight = batch 151 | batch = make(machineInstanceSet) 152 | go func() { 153 | m.maybeCancel(ctx, batchInFlight) 154 | m.ch <- eventBatchDone{} 155 | }() 156 | case <-ctx.Done(): 157 | return 158 | } 159 | } 160 | } 161 | 162 | // maybeCancel cancels a machine if its instance is in a terminal state. We use 163 | // the instance status to recover faster than if we were to rely on the 164 | // keepalive timeout alone. 165 | func (m *T) maybeCancel(ctx context.Context, mis machineInstanceSet) { 166 | ids := make([]*string, 0, len(mis)) 167 | byID := make(map[string]machineInstance, len(mis)) 168 | for mi := range mis { 169 | ids = append(ids, aws.String(mi.instanceID)) 170 | byID[mi.instanceID] = mi 171 | } 172 | input := &ec2.DescribeInstancesInput{InstanceIds: ids} 173 | output, err := m.ec2.DescribeInstancesWithContext(ctx, input) 174 | if err != nil { 175 | log.Printf("describing instances after keepalive failure: %v", err) 176 | return 177 | } 178 | for _, res := range output.Reservations { 179 | for _, inst := range res.Instances { 180 | stateName := aws.StringValue(inst.State.Name) 181 | switch stateName { 182 | case ec2.InstanceStateNameShuttingDown: 183 | fallthrough 184 | case ec2.InstanceStateNameTerminated: 185 | fallthrough 186 | case ec2.InstanceStateNameStopping: 187 | fallthrough 188 | case ec2.InstanceStateNameStopped: 189 | mi, ok := byID[aws.StringValue(inst.InstanceId)] 190 | if !ok { 191 | log.Printf("unexpected instance ID: %s", aws.StringValue(inst.InstanceId)) 192 | continue 193 | } 194 | log.Printf("canceling %s because it is in state '%s'", mi.machine.Addr, stateName) 195 | mi.machine.Cancel() 196 | } 197 | if inst.StateReason != nil { 198 | // TODO(jcharumilind): Account spot failures and shift to 199 | // on-demand instances if there is too much contention in the 200 | // spot market. 201 | log.Printf( 202 | "instance %s: state:%s, code:%s, message:%s", 203 | aws.StringValue(inst.InstanceId), 204 | aws.StringValue(inst.State.Name), 205 | aws.StringValue(inst.StateReason.Code), 206 | aws.StringValue(inst.StateReason.Message), 207 | ) 208 | } 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /local.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigmachine 6 | 7 | import ( 8 | "context" 9 | "crypto/tls" 10 | "fmt" 11 | "io" 12 | "io/ioutil" 13 | "net" 14 | "net/http" 15 | "os" 16 | "os/exec" 17 | "strings" 18 | "sync" 19 | "time" 20 | 21 | "github.com/grailbio/base/config" 22 | "github.com/grailbio/base/errors" 23 | "github.com/grailbio/base/log" 24 | "github.com/grailbio/bigmachine/internal/authority" 25 | bigioutil "github.com/grailbio/bigmachine/internal/ioutil" 26 | "github.com/grailbio/bigmachine/internal/tee" 27 | "golang.org/x/net/http2" 28 | ) 29 | 30 | func init() { 31 | config.Register("bigmachine/local", func(constr *config.Constructor[System]) { 32 | constr.Doc = "bigmachine/local is the bigmachine instance used for local process-based clusters" 33 | constr.New = func() (System, error) { 34 | return Local, nil 35 | } 36 | }) 37 | 38 | config.Default("bigmachine/system", "bigmachine/local") 39 | 40 | RegisterSystem("local", Local) 41 | } 42 | 43 | const maxConcurrentStreams = 20000 44 | 45 | // Local is a System that instantiates machines by 46 | // creating new processes on the local machine. 47 | var Local System = new(localSystem) 48 | 49 | // LocalSystem implements a System that instantiates machines 50 | // by creating processes on the local machine. 51 | type localSystem struct { 52 | Gobable struct{} // to make the struct gob-encodable 53 | 54 | // initOnce is used to guarantee one-time (lazy) initialization of this 55 | // system. 56 | initOnce sync.Once 57 | // initErr holds any error from initialization. 58 | initErr error 59 | 60 | authorityFilename string 61 | authority *authority.T 62 | 63 | mu sync.Mutex 64 | muxers map[*Machine]*tee.Writer 65 | } 66 | 67 | func (*localSystem) Name() string { 68 | return "local" 69 | } 70 | 71 | func (s *localSystem) Init() error { 72 | s.initOnce.Do(func() { 73 | f, err := ioutil.TempFile("", "") 74 | if err != nil { 75 | s.initErr = err 76 | return 77 | } 78 | s.authorityFilename = f.Name() 79 | _ = f.Close() 80 | if err = os.Remove(s.authorityFilename); err != nil { 81 | s.initErr = err 82 | return 83 | } 84 | if s.authority, err = authority.New(s.authorityFilename); err != nil { 85 | s.initErr = err 86 | return 87 | } 88 | s.muxers = make(map[*Machine]*tee.Writer) 89 | }) 90 | return s.initErr 91 | } 92 | 93 | func (s *localSystem) Start(ctx context.Context, _ *B, count int) ([]*Machine, error) { 94 | machines := make([]*Machine, count) 95 | for i := range machines { 96 | port, err := getFreeTCPPort() 97 | if err != nil { 98 | return nil, err 99 | } 100 | cmd := exec.Command(os.Args[0], os.Args[1:]...) 101 | cmd.Env = os.Environ() 102 | cmd.Env = append(cmd.Env, "BIGMACHINE_MODE=machine") 103 | cmd.Env = append(cmd.Env, "BIGMACHINE_SYSTEM=local") 104 | muxer := new(tee.Writer) 105 | cmd.Stdout = muxer 106 | cmd.Stderr = muxer 107 | cmd.Env = append(cmd.Env, fmt.Sprintf("BIGMACHINE_ADDR=localhost:%d", port)) 108 | cmd.Env = append(cmd.Env, fmt.Sprintf("BIGMACHINE_AUTHORITY=%s", s.authorityFilename)) 109 | 110 | m := new(Machine) 111 | m.Addr = fmt.Sprintf("https://localhost:%d/", port) 112 | s.mu.Lock() 113 | s.muxers[m] = muxer 114 | s.mu.Unlock() 115 | m.Maxprocs = 1 116 | err = cmd.Start() 117 | if err != nil { 118 | return nil, err 119 | } 120 | go func() { 121 | if err := cmd.Wait(); err != nil { 122 | log.Printf("machine %s terminated with error: %v", m.Addr, err) 123 | } else { 124 | log.Printf("machine %s terminated", m.Addr) 125 | } 126 | }() 127 | machines[i] = m 128 | } 129 | return machines, nil 130 | } 131 | 132 | func (*localSystem) Main() error { 133 | var c chan struct{} 134 | <-c // hang forever 135 | panic("not reached") 136 | } 137 | 138 | func (s *localSystem) Event(typ string, fieldPairs ...interface{}) { 139 | fields := []string{fmt.Sprintf("eventType:%s", typ)} 140 | for i := 0; i < len(fieldPairs); i++ { 141 | name := fieldPairs[i].(string) 142 | i++ 143 | value := fieldPairs[i] 144 | fields = append(fields, fmt.Sprintf("%s:%v", name, value)) 145 | } 146 | log.Debug.Print(strings.Join(fields, ", ")) 147 | } 148 | 149 | func (s *localSystem) ListenAndServe(addr string, handler http.Handler) error { 150 | if addr == "" { 151 | addr = os.Getenv("BIGMACHINE_ADDR") 152 | } 153 | if addr == "" { 154 | return errors.New("no address defined") 155 | } 156 | if filename := os.Getenv("BIGMACHINE_AUTHORITY"); filename != "" { 157 | s.authorityFilename = filename 158 | var err error 159 | s.authority, err = authority.New(s.authorityFilename) 160 | if err != nil { 161 | return err 162 | } 163 | } 164 | _, config, err := s.authority.HTTPSConfig() 165 | if err != nil { 166 | return err 167 | } 168 | config.ClientAuth = tls.RequireAndVerifyClientCert 169 | server := &http.Server{ 170 | TLSConfig: config, 171 | Addr: addr, 172 | Handler: handler, 173 | } 174 | err = http2.ConfigureServer(server, &http2.Server{ 175 | MaxConcurrentStreams: maxConcurrentStreams, 176 | }) 177 | if err != nil { 178 | return fmt.Errorf("error configuring server: %v", err) 179 | } 180 | return server.ListenAndServeTLS("", "") 181 | } 182 | 183 | func (s *localSystem) HTTPClient() *http.Client { 184 | config, _, err := s.authority.HTTPSConfig() 185 | if err != nil { 186 | // TODO: propagate error, or return error client 187 | log.Fatalf("error build TLS configuration: %v", err) 188 | } 189 | transport := &http.Transport{TLSClientConfig: config} 190 | if err = http2.ConfigureTransport(transport); err != nil { 191 | // TODO: propagate error, or return error client 192 | log.Fatalf("error configuring transport: %v", err) 193 | } 194 | return &http.Client{Transport: transport} 195 | } 196 | 197 | func (*localSystem) Exit(code int) { 198 | os.Exit(code) 199 | } 200 | 201 | func (*localSystem) Shutdown() {} 202 | 203 | func (*localSystem) Maxprocs() int { 204 | return 1 205 | } 206 | 207 | func (*localSystem) KeepaliveConfig() (period, timeout, rpcTimeout time.Duration) { 208 | period = time.Minute 209 | timeout = 2 * time.Minute 210 | rpcTimeout = 10 * time.Second 211 | return 212 | } 213 | 214 | func (s *localSystem) Tail(ctx context.Context, m *Machine) (io.Reader, error) { 215 | s.mu.Lock() 216 | muxer := s.muxers[m] 217 | s.mu.Unlock() 218 | if muxer == nil { 219 | return nil, errors.New("machine not under management") 220 | } 221 | r, w := io.Pipe() 222 | go func() { 223 | cancel := muxer.Tee(w) 224 | <-ctx.Done() 225 | cancel() 226 | w.CloseWithError(ctx.Err()) 227 | }() 228 | return r, nil 229 | } 230 | 231 | func (s *localSystem) Read(ctx context.Context, m *Machine, filename string) (io.Reader, error) { 232 | f, err := os.Open(filename) 233 | if err != nil { 234 | return nil, err 235 | } 236 | return bigioutil.NewClosingReader(f), nil 237 | } 238 | 239 | func (*localSystem) KeepaliveFailed(context.Context, *Machine) {} 240 | 241 | func getFreeTCPPort() (int, error) { 242 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 243 | if err != nil { 244 | return 0, err 245 | } 246 | l, err := net.ListenTCP("tcp", addr) 247 | if err != nil { 248 | return 0, err 249 | } 250 | port := l.Addr().(*net.TCPAddr).Port 251 | l.Close() 252 | return port, nil 253 | } 254 | -------------------------------------------------------------------------------- /cmd/bigpi/bigpi.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Bigpi is an example bigmachine program that estimates digits of Pi 7 | using the Monte Carlo method. It distributes work by instantiating 8 | multiple machines and calling them to make samples, returning the 9 | total number of the samples that fell inside of the unit circle. 10 | 11 | We can run it locally with a small number of sample to test: 12 | 13 | % bigpi -n 1000000 14 | 2018/03/16 15:21:05 waiting for machines to come online 15 | 2018/03/16 15:21:08 machine http://localhost:63880/ RUNNING 16 | 2018/03/16 15:21:08 machine http://localhost:63878/ RUNNING 17 | 2018/03/16 15:21:08 machine http://localhost:63879/ RUNNING 18 | 2018/03/16 15:21:08 machine http://localhost:63881/ RUNNING 19 | 2018/03/16 15:21:08 machine http://localhost:63877/ RUNNING 20 | 2018/03/16 15:21:08 all machines are ready 21 | 2018/03/16 15:21:08 distributing work among 5 cores 22 | http://localhost:63878/: 2018/03/16 15:21:08 0/200000 23 | http://localhost:63880/: 2018/03/16 15:21:08 0/200000 24 | http://localhost:63879/: 2018/03/16 15:21:08 0/200000 25 | http://localhost:63881/: 2018/03/16 15:21:08 0/200000 26 | 2018/03/16 15:21:08 total=784425 nsamples=1000000 27 | π = 3.1377 28 | 29 | By using a large EC2 instance we can distribute the work over 100s of cores 30 | trivially: 31 | 32 | % bigpi -bigsystem ec2 -bigec2type c5.18xlarge -n 1000000000000 33 | 2018/03/20 21:00:05 waiting for machines to come online 34 | 2018/03/20 21:01:09 machine https://ec2-54-213-185-145.us-west-2.compute.amazonaws.com:2000/ RUNNING 35 | 2018/03/20 21:01:09 machine https://ec2-35-164-137-2.us-west-2.compute.amazonaws.com:2000/ RUNNING 36 | 2018/03/20 21:01:09 machine https://ec2-34-208-105-231.us-west-2.compute.amazonaws.com:2000/ RUNNING 37 | 2018/03/20 21:01:09 machine https://ec2-34-211-149-59.us-west-2.compute.amazonaws.com:2000/ RUNNING 38 | 2018/03/20 21:01:09 machine https://ec2-34-223-251-92.us-west-2.compute.amazonaws.com:2000/ RUNNING 39 | 2018/03/20 21:01:09 all machines are ready 40 | 2018/03/20 21:01:09 distributing work among 360 cores 41 | https://ec2-34-208-105-231.us-west-2.compute.amazonaws.com:2000/: 2018/03/20 21:01:09 0/2777777777 42 | https://ec2-34-223-251-92.us-west-2.compute.amazonaws.com:2000/: 2018/03/20 21:01:09 0/2777777777 43 | ... 44 | 2018/03/20 21:13:27 total=785397678380 nsamples=1000000000000 45 | π = 3.141590713520 46 | 47 | Once a bigmachine program is running, we can profile it using the 48 | standard Go pprof tooling. The returned profile is sampled from the 49 | whole cluster and merged. In the first iteration of this program, this helped 50 | find a bug: we were using the global rand.Float64 which requires a lock. 51 | The CPU profile highlighted the lock contention easily: 52 | 53 | % go tool pprof localhost:3333/debug/bigmachine/pprof/profile 54 | Fetching profile over HTTP from http://localhost:3333/debug/bigmachine/pprof/profile 55 | Saved profile in /Users/marius/pprof/pprof.045821636.samples.cpu.001.pb.gz 56 | File: 045821636 57 | Type: cpu 58 | Time: Mar 16, 2018 at 3:17pm (PDT) 59 | Duration: 2.51mins, Total samples = 16.80mins (669.32%) 60 | Entering interactive mode (type "help" for commands, "o" for options) 61 | (pprof) top 62 | Showing nodes accounting for 779.47s, 77.31% of 1008.18s total 63 | Dropped 51 nodes (cum <= 5.04s) 64 | Showing top 10 nodes out of 58 65 | flat flat% sum% cum cum% 66 | 333.11s 33.04% 33.04% 333.11s 33.04% runtime.procyield 67 | 116.71s 11.58% 44.62% 469.55s 46.57% runtime.lock 68 | 76.35s 7.57% 52.19% 347.21s 34.44% sync.(*Mutex).Lock 69 | 65.79s 6.53% 58.72% 65.79s 6.53% runtime.futex 70 | 41.48s 4.11% 62.83% 202.05s 20.04% sync.(*Mutex).Unlock 71 | 34.10s 3.38% 66.21% 364.36s 36.14% runtime.findrunnable 72 | 33s 3.27% 69.49% 33s 3.27% runtime.cansemacquire 73 | 32.72s 3.25% 72.73% 51.01s 5.06% runtime.runqgrab 74 | 24.88s 2.47% 75.20% 57.72s 5.73% runtime.unlock 75 | 21.33s 2.12% 77.31% 21.33s 2.12% math/rand.(*rngSource).Uint64 76 | 77 | */ 78 | package main 79 | 80 | import ( 81 | "context" 82 | "encoding/gob" 83 | "flag" 84 | "fmt" 85 | "math" 86 | "math/big" 87 | "math/rand" 88 | "net/http" 89 | _ "net/http/pprof" 90 | "sync/atomic" 91 | 92 | "github.com/grailbio/base/log" 93 | "github.com/grailbio/bigmachine" 94 | "github.com/grailbio/bigmachine/driver" 95 | "golang.org/x/sync/errgroup" 96 | ) 97 | 98 | func init() { 99 | gob.Register(circlePI{}) 100 | } 101 | 102 | type circlePI struct{} 103 | 104 | // Sample generates n points inside the unit square and reports 105 | // how many of these fall inside the unit circle. 106 | func (circlePI) Sample(ctx context.Context, n uint64, m *uint64) error { 107 | r := rand.New(rand.NewSource(rand.Int63())) 108 | for i := uint64(0); i < n; i++ { 109 | if i%1e7 == 0 { 110 | log.Printf("%d/%d", i, n) 111 | } 112 | x, y := r.Float64(), r.Float64() 113 | if (x-0.5)*(x-0.5)+(y-0.5)*(y-0.5) < 0.25 { 114 | *m++ 115 | } 116 | } 117 | return nil 118 | } 119 | 120 | func main() { 121 | var ( 122 | nsamples = flag.Int("n", 1e10, "number of samples to make") 123 | nmachine = flag.Int("nmach", 5, "number of machines to provision for the task") 124 | ) 125 | log.AddFlags() 126 | flag.Parse() 127 | b := driver.Start() 128 | defer b.Shutdown() 129 | 130 | // Launch a local web server so we have access to profiles. 131 | go func() { 132 | err := http.ListenAndServe(":3333", nil) 133 | log.Printf("http.ListenAndServe: %v", err) 134 | }() 135 | ctx := context.Background() 136 | 137 | // Start the desired number of machines, 138 | // each with the circlePI service. 139 | machines, err := b.Start(ctx, *nmachine, bigmachine.Services{ 140 | "PI": circlePI{}, 141 | }) 142 | if err != nil { 143 | log.Fatal(err) 144 | } 145 | log.Print("waiting for machines to come online") 146 | for _, m := range machines { 147 | <-m.Wait(bigmachine.Running) 148 | log.Printf("machine %s %s", m.Addr, m.State()) 149 | if err := m.Err(); err != nil { 150 | log.Fatal(err) 151 | } 152 | } 153 | log.Print("all machines are ready") 154 | // Number of samples per machine 155 | numPerMachine := uint64(*nsamples) / uint64(*nmachine) 156 | 157 | // Divide the total number of samples among all the processors on 158 | // each machine. Aggregate the counts and then report the estimate. 159 | var total uint64 160 | var cores int 161 | g, ctx := errgroup.WithContext(ctx) 162 | for _, m := range machines { 163 | m := m 164 | for i := 0; i < m.Maxprocs; i++ { 165 | cores++ 166 | g.Go(func() error { 167 | var count uint64 168 | err := m.Call(ctx, "PI.Sample", numPerMachine/uint64(m.Maxprocs), &count) 169 | if err == nil { 170 | atomic.AddUint64(&total, count) 171 | } 172 | return err 173 | }) 174 | } 175 | } 176 | log.Printf("distributing work among %d cores", cores) 177 | if err := g.Wait(); err != nil { 178 | log.Fatal(err) 179 | } 180 | log.Printf("total=%d nsamples=%d", total, *nsamples) 181 | var ( 182 | pi = big.NewRat(int64(4*total), int64(*nsamples)) 183 | prec = int(math.Log(float64(*nsamples)) / math.Log(10)) 184 | ) 185 | fmt.Printf("π = %s\n", pi.FloatString(prec)) 186 | } 187 | -------------------------------------------------------------------------------- /profile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigmachine 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "net/http" 13 | "sort" 14 | "strconv" 15 | "sync" 16 | "time" 17 | 18 | "github.com/google/pprof/profile" 19 | "github.com/grailbio/base/diagnostic/dump" 20 | "github.com/grailbio/base/log" 21 | "github.com/grailbio/bigmachine/internal/filebuf" 22 | "golang.org/x/sync/errgroup" 23 | ) 24 | 25 | // ProfileHandler implements an HTTP handler for a profile. The 26 | // handler gathers profiles from all machines (at the time of 27 | // collection) and returns a merged profile representing all cluster 28 | // activity. 29 | type profileHandler struct { 30 | b *B 31 | which string 32 | } 33 | 34 | func (h *profileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 35 | var ( 36 | sec64, _ = strconv.ParseInt(r.FormValue("seconds"), 10, 64) 37 | sec = int(sec64) 38 | debug, _ = strconv.Atoi(r.FormValue("debug")) 39 | gc, _ = strconv.Atoi(r.FormValue("gc")) 40 | addr = r.FormValue("machine") 41 | ) 42 | p := profiler{ 43 | b: h.b, 44 | which: h.which, 45 | addr: addr, 46 | sec: sec, 47 | debug: debug, 48 | gc: gc > 0, 49 | } 50 | w.Header().Set("Content-Type", p.ContentType()) 51 | err := p.Marshal(r.Context(), w) 52 | if err != nil { 53 | code := http.StatusInternalServerError 54 | if err == errNoProfiles { 55 | code = http.StatusNotFound 56 | } 57 | profileErrorf(w, code, err.Error()) 58 | } 59 | } 60 | 61 | func getProfile(ctx context.Context, m *Machine, which string, sec, debug int, gc bool) (rc io.ReadCloser, err error) { 62 | if which == "profile" { 63 | err = m.Call(ctx, "Supervisor.CPUProfile", time.Duration(sec)*time.Second, &rc) 64 | } else { 65 | err = m.Call(ctx, "Supervisor.Profile", profileRequest{which, debug, gc}, &rc) 66 | } 67 | return 68 | } 69 | 70 | func profileErrorf(w http.ResponseWriter, code int, message string, args ...interface{}) { 71 | w.Header().Set("Content-Type", "text/plain; charset=utf-8") 72 | w.Header().Set("X-Go-Pprof", "1") 73 | w.WriteHeader(code) 74 | if _, err := fmt.Fprintf(w, message, args...); err != nil { 75 | log.Printf("error writing profile 500: %v", err) 76 | } 77 | } 78 | 79 | // profiler writes (possibly aggregated) profiles, configured by its fields. 80 | type profiler struct { 81 | b *B 82 | // which is the name of the profile to write. 83 | which string 84 | // addr is the specific machine's profile to write. If addr == "", profiles 85 | // are aggregated from all of b's machines. 86 | addr string 87 | // sec is the number of seconds for which to generate a CPU profile. It is 88 | // only relevant when which == "profile". 89 | sec int 90 | // debug is the debug value passed to pprof to determine the format of the 91 | // profile output. See pprof documentation for details. 92 | debug int 93 | // gc determines whether we request a garbage collection before taking the 94 | // profile. This is only relevant when which == "heap". 95 | gc bool 96 | } 97 | 98 | // errNoProfiles is the error that is returned by (profiler).Marshal when there 99 | // are no profiles from the cluster machines. We use this to signal that we 100 | // want to return a StatusNotFound when we are writing the profile in an HTTP 101 | // response. 102 | var errNoProfiles = errors.New("no profiles are available at this time") 103 | 104 | // ContentType returns the expected content type, assuming success, of a call 105 | // to Marshal. This is used to set the Content-Type header when we are writing 106 | // the profile in an HTTP response. This may be overridden if there is an 107 | // error. 108 | func (p profiler) ContentType() string { 109 | if p.debug > 0 && p.which != "profile" { 110 | return "text/plain; charset=utf-8" 111 | } 112 | return "application/octet-stream" 113 | } 114 | 115 | // Marshal writes the profile configured in pw to w. 116 | func (p profiler) Marshal(ctx context.Context, w io.Writer) (err error) { 117 | if p.addr != "" { 118 | var m *Machine 119 | if m, err = p.b.Dial(ctx, p.addr); err != nil { 120 | return fmt.Errorf("dialing machine: %v", err) 121 | } 122 | var rc io.ReadCloser 123 | if rc, err = getProfile(ctx, m, p.which, p.sec, p.debug, p.gc); err != nil { 124 | return fmt.Errorf("collecting %s profile: %v", p.which, err) 125 | } 126 | defer func() { 127 | cerr := rc.Close() 128 | if err == nil { 129 | err = cerr 130 | } 131 | }() 132 | _, err = io.Copy(w, rc) 133 | if err != nil { 134 | return fmt.Errorf("writing %s profile: %v", p.which, err) 135 | } 136 | return nil 137 | } 138 | g, ctx := errgroup.WithContext(ctx) 139 | var ( 140 | mu sync.Mutex 141 | profiles = make(map[*Machine]io.ReadCloser) 142 | machines = p.b.Machines() 143 | ) 144 | for _, m := range machines { 145 | if m.State() != Running { 146 | continue 147 | } 148 | m := m 149 | g.Go(func() (err error) { 150 | rc, err := getProfile(ctx, m, p.which, p.sec, p.debug, p.gc) 151 | if err != nil { 152 | log.Printf("collecting %s from %s: %v", p.which, m.Addr, err) 153 | return nil 154 | } 155 | prof, err := filebuf.New(rc) 156 | if err != nil { 157 | log.Printf("reading profile from %s: %v", m.Addr, err) 158 | return nil 159 | } 160 | mu.Lock() 161 | profiles[m] = prof 162 | mu.Unlock() 163 | return nil 164 | }) 165 | } 166 | defer func() { 167 | // We generally close the profile buffers as we are done using them to 168 | // free resources, setting the corresponding entries in profiles to nil. 169 | // In error cases, we may still have buffers left to close. We do that 170 | // here. 171 | for _, prof := range profiles { 172 | if prof == nil { 173 | continue 174 | } 175 | _ = prof.Close() 176 | } 177 | }() 178 | if err = g.Wait(); err != nil { 179 | return fmt.Errorf("fetching profiles: %v", err) 180 | } 181 | if len(profiles) == 0 { 182 | return errNoProfiles 183 | } 184 | // Debug output is intended for human consumption. 185 | if p.debug > 0 && p.which != "profiles" { 186 | sort.Slice(machines, func(i, j int) bool { return machines[i].Addr < machines[j].Addr }) 187 | for _, m := range machines { 188 | prof := profiles[m] 189 | if prof == nil { 190 | continue 191 | } 192 | fmt.Fprintf(w, "%s:\n", m.Addr) 193 | _, err = io.Copy(w, prof) 194 | _ = prof.Close() 195 | profiles[m] = nil 196 | if err != nil { 197 | return fmt.Errorf("appending profile from %s: %v", m.Addr, err) 198 | } 199 | fmt.Fprintln(w) 200 | } 201 | return nil 202 | } 203 | 204 | var parsed []*profile.Profile 205 | for m, rc := range profiles { 206 | var prof *profile.Profile 207 | prof, err = profile.Parse(rc) 208 | _ = rc.Close() 209 | profiles[m] = nil 210 | if err != nil { 211 | return fmt.Errorf("parsing profile from %s: %v", m.Addr, err) 212 | } 213 | parsed = append(parsed, prof) 214 | } 215 | prof, err := profile.Merge(parsed) 216 | if err != nil { 217 | return fmt.Errorf("merging profiles: %v", err) 218 | } 219 | if err := prof.Write(w); err != nil { 220 | return fmt.Errorf("writing profile: %v", err) 221 | } 222 | return nil 223 | } 224 | 225 | func makeProfileDumpFunc(b *B, which string, debug int) dump.Func { 226 | p := profiler{ 227 | b: b, 228 | which: which, 229 | sec: 30, 230 | debug: debug, 231 | gc: false, 232 | } 233 | return func(ctx context.Context, w io.Writer) error { 234 | err := p.Marshal(ctx, w) 235 | if err == errNoProfiles { 236 | return dump.ErrSkipPart 237 | } 238 | return err 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /machine_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigmachine 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "encoding/gob" 11 | "io" 12 | "io/ioutil" 13 | "log" 14 | "net/http" 15 | "net/http/httptest" 16 | "runtime" 17 | "testing" 18 | "time" 19 | 20 | "github.com/grailbio/base/errors" 21 | "github.com/grailbio/bigmachine/rpc" 22 | ) 23 | 24 | var fakeDigest = digester.FromString("fake binary") 25 | 26 | type fakeSupervisor struct { 27 | Args []string 28 | Environ []string 29 | Image []byte 30 | LastKeepalive time.Time 31 | Hung bool 32 | Execd bool 33 | } 34 | 35 | func (s *fakeSupervisor) Setenv(ctx context.Context, env []string, _ *struct{}) error { 36 | s.Environ = env 37 | return nil 38 | } 39 | 40 | func (s *fakeSupervisor) Setargs(ctx context.Context, args []string, _ *struct{}) error { 41 | s.Args = args 42 | return nil 43 | } 44 | 45 | func (s *fakeSupervisor) Setbinary(ctx context.Context, binary io.Reader, _ *struct{}) (err error) { 46 | s.Image, err = ioutil.ReadAll(binary) 47 | return err 48 | } 49 | 50 | func (s *fakeSupervisor) GetBinary(ctx context.Context, _ struct{}, rc *io.ReadCloser) error { 51 | if s.Image == nil { 52 | return errors.E(errors.Invalid, "no binary set") 53 | } 54 | *rc = ioutil.NopCloser(bytes.NewReader(s.Image)) 55 | return nil 56 | } 57 | 58 | func (s *fakeSupervisor) Exec(ctx context.Context, exec io.Reader, _ *struct{}) error { 59 | s.Execd = true 60 | return nil 61 | } 62 | 63 | func (s *fakeSupervisor) Tail(ctx context.Context, fd int, rc *io.ReadCloser) error { 64 | return errors.New("not supported") 65 | } 66 | 67 | func (s *fakeSupervisor) Ping(ctx context.Context, seq int, replyseq *int) error { 68 | *replyseq = seq 69 | return nil 70 | } 71 | 72 | func (s *fakeSupervisor) Info(ctx context.Context, _ struct{}, info *Info) error { 73 | info.Goos = runtime.GOOS 74 | info.Goarch = runtime.GOARCH 75 | info.Digest = fakeDigest 76 | return nil 77 | } 78 | 79 | func (s *fakeSupervisor) Keepalive(ctx context.Context, next time.Duration, reply *keepaliveReply) error { 80 | if s.Hung { 81 | <-ctx.Done() 82 | return ctx.Err() 83 | } 84 | s.LastKeepalive = time.Now() 85 | reply.Next = next 86 | reply.Healthy = true 87 | return nil 88 | } 89 | 90 | func (s *fakeSupervisor) Hang(ctx context.Context, _ struct{}, _ *struct{}) error { 91 | <-ctx.Done() 92 | return ctx.Err() 93 | } 94 | 95 | func (s *fakeSupervisor) Register(ctx context.Context, svc service, _ *struct{}) error { 96 | // Tests only require that we Init services (if needed), so we don't do any 97 | // actual registration. 98 | return maybeInit(svc.Instance, nil) 99 | } 100 | 101 | func newTestMachine(t *testing.T, params ...Param) (m *Machine, supervisor *fakeSupervisor, shutdown func()) { 102 | t.Helper() 103 | supervisor = new(fakeSupervisor) 104 | srv := rpc.NewServer() 105 | if err := srv.Register("Supervisor", supervisor); err != nil { 106 | t.Fatal(err) 107 | } 108 | httpsrv := httptest.NewServer(srv) 109 | client, err := rpc.NewClient(func() *http.Client { return httpsrv.Client() }, "/") 110 | if err != nil { 111 | httpsrv.Close() 112 | t.Fatal(err) 113 | } 114 | m = &Machine{ 115 | Addr: httpsrv.URL, 116 | client: client, 117 | owner: true, 118 | keepalivePeriod: time.Minute, 119 | keepaliveTimeout: 2 * time.Minute, 120 | keepaliveRpcTimeout: 10 * time.Second, 121 | tailDone: make(chan struct{}), 122 | } 123 | for _, param := range params { 124 | param.applyParam(m) 125 | } 126 | m.start(nil) 127 | return m, supervisor, func() { 128 | m.Cancel() 129 | select { 130 | case <-m.Wait(Stopped): 131 | case <-time.After(time.Second): 132 | t.Log("failed to stop server after 1 second") 133 | } 134 | httpsrv.Close() 135 | } 136 | } 137 | 138 | func TestMachineBootup(t *testing.T) { 139 | m, supervisor, shutdown := newTestMachine(t) 140 | defer shutdown() 141 | 142 | <-m.Wait(Running) 143 | if got, want := m.State(), Running; got != want { 144 | t.Errorf("got %v, want %v", got, want) 145 | } 146 | r, err := binary() 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | image, err := ioutil.ReadAll(r) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | if !bytes.Equal(supervisor.Image, image) { 155 | t.Error("image does not match") 156 | } 157 | if time.Since(supervisor.LastKeepalive) > time.Minute { 158 | t.Errorf("failed to maintain keepalive") 159 | } 160 | } 161 | 162 | func TestMachineEnv(t *testing.T) { 163 | m, supervisor, shutdown := newTestMachine(t, Environ{"test=yes"}) 164 | defer shutdown() 165 | <-m.Wait(Running) 166 | if got, want := len(supervisor.Environ), 1; got != want { 167 | t.Fatalf("got %v, want %v", got, want) 168 | } 169 | if got, want := supervisor.Environ[0], "test=yes"; got != want { 170 | t.Errorf("got %v, want %v", got, want) 171 | } 172 | } 173 | 174 | func TestCallTimeout(t *testing.T) { 175 | m, _, shutdown := newTestMachine(t) 176 | defer shutdown() 177 | const timeout = 2 * time.Second 178 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 179 | err := m.Call(ctx, "Supervisor.Hang", struct{}{}, nil) 180 | if got, want := err, context.DeadlineExceeded; got != want { 181 | t.Errorf("got %v, want %v", got, want) 182 | } 183 | cancel() 184 | } 185 | 186 | func TestMachineContext(t *testing.T) { 187 | log.SetFlags(log.Llongfile) 188 | m, supervisor, shutdown := newTestMachine(t) 189 | defer shutdown() 190 | supervisor.Hung = true 191 | ctx, cancel := context.WithCancel(context.Background()) 192 | go func() { 193 | time.Sleep(5 * time.Second) 194 | cancel() 195 | }() 196 | err := m.Call(ctx, "Supervisor.Hang", struct{}{}, nil) 197 | if got, want := err, context.Canceled; got != want { 198 | t.Errorf("got %v, want %v", got, want) 199 | } 200 | } 201 | 202 | // serviceGobUnregistered is a service that is not registered with gob, so 203 | // attempts to register it will fail. 204 | type serviceGobUnregistered struct{} 205 | 206 | // TestServiceGobUnregisteredFastFail verifies that we fail fast when a service 207 | // is not gob-encodable. 208 | func TestServiceGobUnregisteredFastFail(t *testing.T) { 209 | m, _, shutdown := newTestMachine(t, Services{"GobUnregistered": serviceGobUnregistered{}}) 210 | defer shutdown() 211 | select { 212 | case <-m.Wait(Running): 213 | if m.State() == Running { 214 | t.Fatalf("machine is running with broken service") 215 | } 216 | case <-time.After(2 * time.Minute): 217 | // If our test environment causes this to falsely fail, we almost 218 | // surely have lots of other problems, as this should otherwise fail 219 | // almost instantly. 220 | t.Fatalf("took too long to fail") 221 | } 222 | } 223 | 224 | // serviceInitPanic is a service that panics in Init, indicating that the 225 | // service is fatally broken. 226 | type serviceInitPanic struct{} 227 | 228 | func (serviceInitPanic) Init(b *B) error { 229 | panic("") 230 | } 231 | 232 | func init() { 233 | gob.Register(serviceInitPanic{}) 234 | } 235 | 236 | // TestBadServiceFastFail verifies that we fail fast when a service panics in 237 | // its Init. 238 | func TestServiceInitPanicFastFail(t *testing.T) { 239 | m, _, shutdown := newTestMachine(t, Services{"InitPanic": serviceInitPanic{}}) 240 | defer shutdown() 241 | select { 242 | case <-m.Wait(Running): 243 | if m.State() == Running { 244 | t.Fatalf("machine is running with broken service") 245 | } 246 | case <-time.After(2 * time.Minute): 247 | // If our test environment causes this to falsely fail, we almost 248 | // surely have lots of other problems, as this should otherwise fail 249 | // almost instantly. 250 | t.Fatalf("took too long to fail") 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /internal/authority/authority.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package authority provides an in-process TLS certificate authority, 6 | // useful for creating and distributing TLS certificates for mutually authenticated 7 | // HTTPS networking within Bigmachine. 8 | package authority 9 | 10 | import ( 11 | "bytes" 12 | "crypto/rand" 13 | "crypto/rsa" 14 | "crypto/tls" 15 | "crypto/x509" 16 | "crypto/x509/pkix" 17 | "encoding/pem" 18 | "errors" 19 | "fmt" 20 | "io/ioutil" 21 | "math/big" 22 | "net" 23 | "os" 24 | "time" 25 | ) 26 | 27 | // DriftMargin is the amount of acceptable clock drift during 28 | // certificate issuing and verification. 29 | const DriftMargin = time.Minute 30 | 31 | // CertDuration is the duration of cert validity for the certificates 32 | // issued by authorities. 33 | const certDuration = 7 * 24 * time.Hour 34 | 35 | // A T is a TLS certificate authority which can issue client and server 36 | // certificates and provide configuration for HTTPS clients. 37 | type T struct { 38 | key *rsa.PrivateKey 39 | cert *x509.Certificate 40 | 41 | // The CA certificate and key are stored in PEM-encoded bytes 42 | // as most of the Go APIs operate directly on these. 43 | certPEM, keyPEM []byte 44 | } 45 | 46 | // New creates a new certificate authority, reading the PEM-encoded 47 | // certificate and private key from the provided path. If the path 48 | // does not exist, newCA instead creates a new certificate authority 49 | // and stores it at the provided path. If path is empty, the 50 | // authority is ephemeral. 51 | func New(filename string) (*T, error) { 52 | // As an extra precaution, we always exercise the read path, so if 53 | // the CA PEM is missing, we generate it, and then read it back. 54 | pemBlock, err := cached(filename, func() ([]byte, error) { 55 | key, err := rsa.GenerateKey(rand.Reader, 2048) 56 | if err != nil { 57 | return nil, err 58 | } 59 | template := x509.Certificate{ 60 | SerialNumber: big.NewInt(1), 61 | Subject: pkix.Name{CommonName: "bigmachine"}, 62 | NotBefore: time.Now().Add(-DriftMargin), 63 | // Newton says we have at least this long: 64 | // https://newtonprojectca.files.wordpress.com/2013/06/reply-to-tom-harpur-2-page-full-version.pdf 65 | NotAfter: time.Date(2060, 1, 1, 0, 0, 0, 0, time.UTC), 66 | 67 | KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, 68 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, 69 | BasicConstraintsValid: true, 70 | IsCA: true, 71 | } 72 | cert, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) 73 | if err != nil { 74 | return nil, err 75 | } 76 | var b bytes.Buffer 77 | // Save it also. 78 | if err := pem.Encode(&b, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil { 79 | return nil, err 80 | } 81 | if err := pem.Encode(&b, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}); err != nil { 82 | return nil, err 83 | } 84 | return b.Bytes(), nil 85 | }) 86 | if err != nil { 87 | return nil, fmt.Errorf("could not build CA: %v", err) 88 | } 89 | 90 | var certBlock, keyBlock []byte 91 | for { 92 | var derBlock *pem.Block 93 | derBlock, pemBlock = pem.Decode(pemBlock) 94 | if derBlock == nil { 95 | break 96 | } 97 | switch derBlock.Type { 98 | case "CERTIFICATE": 99 | certBlock = derBlock.Bytes 100 | case "RSA PRIVATE KEY": 101 | keyBlock = derBlock.Bytes 102 | } 103 | } 104 | 105 | if certBlock == nil || keyBlock == nil { 106 | return nil, errors.New("httpsca: incomplete certificate") 107 | } 108 | ca := new(T) 109 | ca.cert, err = x509.ParseCertificate(certBlock) 110 | if err != nil { 111 | return nil, err 112 | } 113 | ca.key, err = x509.ParsePKCS1PrivateKey(keyBlock) 114 | if err != nil { 115 | return nil, err 116 | } 117 | ca.certPEM, err = encodePEM(&pem.Block{Type: "CERTIFICATE", Bytes: certBlock}) 118 | if err != nil { 119 | return nil, err 120 | } 121 | ca.keyPEM, err = encodePEM(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(ca.key)}) 122 | if err != nil { 123 | return nil, err 124 | } 125 | return ca, nil 126 | } 127 | 128 | // Contents returns the PEM-encoded certificate and key of the authority. 129 | func (c *T) Contents() []byte { 130 | var contents []byte 131 | contents = append(contents, c.certPEM...) 132 | contents = append(contents, c.keyPEM...) 133 | return contents 134 | } 135 | 136 | // Cert returns the authority's x509 certificate. 137 | func (c *T) Cert() *x509.Certificate { 138 | return c.cert 139 | } 140 | 141 | // Issue issues a new certificate out of this CA with the provided common name, ttl, ips, and DNSes. 142 | func (c *T) Issue(cn string, ttl time.Duration, ips []net.IP, dnss []string) ([]byte, *rsa.PrivateKey, error) { 143 | maxSerial := new(big.Int).Lsh(big.NewInt(1), 128) 144 | serial, err := rand.Int(rand.Reader, maxSerial) 145 | if err != nil { 146 | return nil, nil, err 147 | } 148 | now := time.Now().Add(-DriftMargin) 149 | template := x509.Certificate{ 150 | SerialNumber: serial, 151 | Subject: pkix.Name{ 152 | CommonName: cn, 153 | }, 154 | NotBefore: now, 155 | NotAfter: now.Add(DriftMargin + ttl), 156 | KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, 157 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, 158 | BasicConstraintsValid: true, 159 | } 160 | template.IPAddresses = append(template.IPAddresses, ips...) 161 | template.DNSNames = append(template.DNSNames, dnss...) 162 | key, err := rsa.GenerateKey(rand.Reader, 2048) 163 | if err != nil { 164 | return nil, nil, err 165 | } 166 | cert, err := x509.CreateCertificate(rand.Reader, &template, c.cert, &key.PublicKey, c.key) 167 | if err != nil { 168 | return nil, nil, err 169 | } 170 | return cert, key, nil 171 | } 172 | 173 | // HTTPSConfig returns a tls configs based on newly issued TLS certificates from this CA. 174 | func (c *T) HTTPSConfig() (client, server *tls.Config, err error) { 175 | cert, key, err := c.Issue("bigmachine", certDuration, nil, nil) 176 | if err != nil { 177 | return nil, nil, err 178 | } 179 | pool := x509.NewCertPool() 180 | pool.AppendCertsFromPEM(c.certPEM) 181 | 182 | // Load the newly created certificate. 183 | certPEM, err := encodePEM(&pem.Block{Type: "CERTIFICATE", Bytes: cert}) 184 | if err != nil { 185 | return nil, nil, err 186 | } 187 | keyPEM, err := encodePEM(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) 188 | if err != nil { 189 | return nil, nil, err 190 | } 191 | tlscert, err := tls.X509KeyPair(certPEM, keyPEM) 192 | if err != nil { 193 | return nil, nil, err 194 | } 195 | clientConfig := &tls.Config{ 196 | RootCAs: pool, 197 | InsecureSkipVerify: true, 198 | Certificates: []tls.Certificate{tlscert}, 199 | } 200 | serverConfig := &tls.Config{ 201 | ClientCAs: pool, 202 | Certificates: []tls.Certificate{tlscert}, 203 | } 204 | return clientConfig, serverConfig, nil 205 | } 206 | 207 | func encodePEM(block *pem.Block) ([]byte, error) { 208 | var w bytes.Buffer 209 | if err := pem.Encode(&w, block); err != nil { 210 | return nil, err 211 | } 212 | return w.Bytes(), nil 213 | } 214 | 215 | func cached(filename string, gen func() ([]byte, error)) ([]byte, error) { 216 | if filename == "" { 217 | return gen() 218 | } 219 | p, err := ioutil.ReadFile(filename) 220 | if err == nil || !os.IsNotExist(err) { 221 | return p, nil 222 | } 223 | p, err = gen() 224 | if err != nil { 225 | return nil, err 226 | } 227 | return p, ioutil.WriteFile(filename, p, 0600) 228 | } 229 | -------------------------------------------------------------------------------- /testsystem/testsystem.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package testsystem implements a bigmachine system that's useful 6 | // for testing. Unlike other system implementations, 7 | // testsystem.System does not spawn new processes: instead, machines 8 | // are launched inside of the same process. 9 | package testsystem 10 | 11 | import ( 12 | "context" 13 | "encoding/gob" 14 | "io" 15 | "math/rand" 16 | "net/http" 17 | "net/http/httptest" 18 | "os" 19 | "sync" 20 | "time" 21 | 22 | "github.com/grailbio/base/errors" 23 | "github.com/grailbio/bigmachine" 24 | "github.com/grailbio/bigmachine/internal/ioutil" 25 | "github.com/grailbio/bigmachine/rpc" 26 | ) 27 | 28 | func init() { 29 | gob.Register(new(System)) 30 | } 31 | 32 | type closeIdleTransport interface { 33 | CloseIdleConnections() 34 | } 35 | 36 | type machine struct { 37 | *bigmachine.Machine 38 | Cancel func() 39 | Server *httptest.Server 40 | } 41 | 42 | func (m *machine) Kill() { 43 | m.Cancel() 44 | m.Server.CloseClientConnections() 45 | m.Server.Close() 46 | m.Server.Listener.Close() 47 | m.Server.Config.SetKeepAlivesEnabled(false) 48 | } 49 | 50 | // System implements a bigmachine System for testing. 51 | // Systems should be instantiated with New(). 52 | type System struct { 53 | // Machineprocs is the number of procs per machine. 54 | Machineprocs int 55 | 56 | // The following can optionally be specified to customize the behavior 57 | // of Bigmachine's keepalive mechanism. 58 | KeepalivePeriod, KeepaliveTimeout, KeepaliveRpcTimeout time.Duration 59 | 60 | done chan struct{} 61 | exited bool 62 | 63 | client *http.Client 64 | 65 | mu sync.Mutex 66 | cond *sync.Cond 67 | machines []*machine 68 | } 69 | 70 | // New creates a new System that is ready for use. 71 | func New() *System { 72 | s := &System{ 73 | Machineprocs: 1, 74 | done: make(chan struct{}), 75 | client: &http.Client{Transport: &http.Transport{}}, 76 | } 77 | s.cond = sync.NewCond(&s.mu) 78 | return s 79 | } 80 | 81 | // Wait returns the number of live machines in the test system, blocking until 82 | // there are at least n. 83 | func (s *System) Wait(n int) int { 84 | s.mu.Lock() 85 | defer s.mu.Unlock() 86 | for len(s.machines) < n { 87 | s.cond.Wait() 88 | } 89 | return n 90 | } 91 | 92 | // N returns the number of live machines in the test system. 93 | func (s *System) N() int { 94 | s.mu.Lock() 95 | defer s.mu.Unlock() 96 | return len(s.machines) 97 | } 98 | 99 | // Index returns the i'th bigmachine in the system. Index 100 | // panics if the index is out of range. 101 | func (s *System) Index(i int) *bigmachine.Machine { 102 | s.mu.Lock() 103 | defer s.mu.Unlock() 104 | return s.machines[i].Machine 105 | } 106 | 107 | // Machines returns a snapshot of the live machines in the test system. 108 | func (s *System) Machines() []*bigmachine.Machine { 109 | s.mu.Lock() 110 | defer s.mu.Unlock() 111 | snapshot := make([]*bigmachine.Machine, len(s.machines)) 112 | for i, m := range s.machines { 113 | snapshot[i] = m.Machine 114 | } 115 | return snapshot 116 | } 117 | 118 | // Kill kills the machine m that is under management of this system, 119 | // returning true if successful. If m is nil, a random machine is chosen. 120 | func (s *System) Kill(m *bigmachine.Machine) bool { 121 | s.mu.Lock() 122 | defer s.mu.Unlock() 123 | if len(s.machines) == 0 { 124 | return false 125 | } 126 | if m == nil { 127 | m = s.machines[rand.Intn(len(s.machines))].Machine 128 | } 129 | for i, sm := range s.machines { 130 | if sm.Machine == m { 131 | s.machines = append(s.machines[:i], s.machines[i+1:]...) 132 | sm.Kill() 133 | return true 134 | } 135 | } 136 | return false 137 | } 138 | 139 | // Exited tells whether exit has been called on (any) machine. 140 | func (s *System) Exited() bool { 141 | return s.exited 142 | } 143 | 144 | // Shutdown tears down temporary resources allocated by this 145 | // System. 146 | func (s *System) Shutdown() { 147 | close(s.done) 148 | s.mu.Lock() 149 | for _, m := range s.machines { 150 | m.Kill() 151 | } 152 | s.mu.Unlock() 153 | if t, ok := http.DefaultTransport.(closeIdleTransport); ok { 154 | t.CloseIdleConnections() 155 | } 156 | if t, ok := s.client.Transport.(closeIdleTransport); ok { 157 | t.CloseIdleConnections() 158 | } 159 | } 160 | 161 | // Name returns the name of the system. 162 | func (s *System) Name() string { 163 | return "testsystem" 164 | } 165 | 166 | func (*System) Init() error { 167 | return nil 168 | } 169 | 170 | // Main panics. It should not be called, provided a correct 171 | // bigmachine implementation. 172 | func (s *System) Main() error { 173 | panic("Main called on testsystem") 174 | } 175 | 176 | // Event is a no-op for the test system, as we do not care about event logs in 177 | // tests. 178 | func (*System) Event(_ string, _ ...interface{}) {} 179 | 180 | // HTTPClient returns an http.Client that can converse with 181 | // servers created by this test system. 182 | func (s *System) HTTPClient() *http.Client { 183 | return s.client 184 | } 185 | 186 | // ListenAndServe panics. It should not be called, provided a 187 | // correct bigmachine implementation. 188 | func (s *System) ListenAndServe(addr string, handler http.Handler) error { 189 | panic("ListenAndServe called on testsystem") 190 | } 191 | 192 | // Start starts and returns a new Machine. Each new machine is 193 | // provided with a supervisor. The only difference between the 194 | // behavior of a supervisor of a test machine and a regular machine 195 | // is that the test machine supervisor does not exec the process, as 196 | // this would break testing. 197 | func (s *System) Start( 198 | _ context.Context, b *bigmachine.B, count int, 199 | ) ([]*bigmachine.Machine, error) { 200 | s.mu.Lock() 201 | machines := make([]*bigmachine.Machine, count) 202 | for i := range machines { 203 | ctx, cancel := context.WithCancel(context.Background()) 204 | server := rpc.NewServer() 205 | supervisor := bigmachine.StartSupervisor(ctx, b, s, server) 206 | if err := server.Register("Supervisor", supervisor); err != nil { 207 | // Something is broken if we can't register the supervisor in the 208 | // testsystem. 209 | panic(err) 210 | } 211 | mux := http.NewServeMux() 212 | mux.Handle(bigmachine.RpcPrefix, server) 213 | httpServer := httptest.NewServer(mux) 214 | m := &bigmachine.Machine{ 215 | Addr: httpServer.URL, 216 | Maxprocs: s.Machineprocs, 217 | NoExec: true, 218 | } 219 | s.machines = append(s.machines, &machine{m, cancel, httpServer}) 220 | machines[i] = m 221 | } 222 | s.cond.Broadcast() 223 | s.mu.Unlock() 224 | return machines, nil 225 | } 226 | 227 | // Exit marks the system as exited. 228 | func (s *System) Exit(int) { 229 | s.exited = true 230 | } 231 | 232 | // Maxprocs returns 1. 233 | func (s *System) Maxprocs() int { 234 | return s.Machineprocs 235 | } 236 | 237 | func (s *System) KeepaliveConfig() (period, timeout, rpcTimeout time.Duration) { 238 | if period = s.KeepalivePeriod; period == 0 { 239 | period = time.Minute 240 | } 241 | if timeout = s.KeepaliveTimeout; timeout == 0 { 242 | timeout = 2 * time.Minute 243 | } 244 | if rpcTimeout = s.KeepaliveRpcTimeout; rpcTimeout == 0 { 245 | rpcTimeout = 10 * time.Second 246 | } 247 | return 248 | } 249 | 250 | func (s *System) Tail(ctx context.Context, m *bigmachine.Machine) (io.Reader, error) { 251 | return nil, errors.E(errors.NotSupported) 252 | } 253 | 254 | func (s *System) Read(ctx context.Context, m *bigmachine.Machine, filename string) (io.Reader, error) { 255 | f, err := os.Open(filename) 256 | if err != nil { 257 | return nil, err 258 | } 259 | return ioutil.NewClosingReader(f), nil 260 | } 261 | 262 | func (s *System) KeepaliveFailed(context.Context, *bigmachine.Machine) {} 263 | -------------------------------------------------------------------------------- /docs/bootstrap.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | Produced by OmniGraffle 7.12 27 | 2019-10-30 21:10:33 +0000 28 | 29 | 30 | Canvas 1 31 | 32 | Layer 1 33 | 34 | 35 | Bigmachine bootstrap process 36 | 37 | 38 | 39 | 40 | 41 | 42 | Driver 43 | (Go program) 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | Cloud Queue 57 | 58 | 59 | 60 | 61 | Cloud provider 62 | 63 | 64 | 65 | 66 | Server 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 1. request machine 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 3. machine allocated 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 2. allocate machine 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 4. upload self 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 5. exec self binary 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 6. maintain keepalives 130 | 131 | 132 | 133 | 134 | 135 | -------------------------------------------------------------------------------- /rpc/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package rpc 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "encoding/gob" 11 | "fmt" 12 | "io" 13 | "io/ioutil" 14 | "net/http" 15 | "strings" 16 | "sync" 17 | "time" 18 | 19 | "github.com/grailbio/base/errors" 20 | "github.com/grailbio/base/limitbuf" 21 | "github.com/grailbio/base/log" 22 | "golang.org/x/net/context/ctxhttp" 23 | "golang.org/x/time/rate" 24 | ) 25 | 26 | const ( 27 | gobContentType = "application/x-gob" 28 | 29 | // We warn on RPC payloads above this size. 30 | largeRpcPayload = 64 << 20 31 | ) 32 | 33 | // Loggers used to inform the user of large payloads, but without 34 | // spamming them. 35 | var ( 36 | largeArgLogger = &rateLimitingOutputter{rate.NewLimiter(rate.Every(time.Minute), 2), log.GetOutputter()} 37 | largeReplyLogger = &rateLimitingOutputter{rate.NewLimiter(rate.Every(time.Minute), 2), log.GetOutputter()} 38 | ) 39 | 40 | // clientState stores the state of a single client to a single server; 41 | // used to reset client connections when needed. 42 | type clientState struct { 43 | addr string 44 | factory func() *http.Client 45 | 46 | once sync.Once 47 | cached *http.Client 48 | } 49 | 50 | func (c *clientState) init() { 51 | c.cached = c.factory() 52 | } 53 | 54 | func (c *clientState) Client() *http.Client { 55 | c.once.Do(c.init) 56 | return c.cached 57 | } 58 | 59 | // A Client invokes remote methods on RPC servers. 60 | type Client struct { 61 | factory func() *http.Client 62 | prefix string 63 | 64 | // Loggers contains a rate limiting logger per client; 65 | // use getLogger to retrieve it. 66 | loggers sync.Map // map[string]*rateLimitingOutputter 67 | 68 | mu sync.Mutex 69 | clients map[string]*clientState 70 | } 71 | 72 | // NewClient creates a new RPC client. clientFactory is called to create a new 73 | // http.Client object. It may be called repeatedly and concurrently. prefix is 74 | // prepended to the service method when constructing an URL. 75 | func NewClient(clientFactory func() *http.Client, prefix string) (*Client, error) { 76 | return &Client{ 77 | factory: clientFactory, 78 | prefix: prefix, 79 | clients: make(map[string]*clientState), 80 | }, nil 81 | } 82 | 83 | func (c *Client) getClient(addr string) *clientState { 84 | c.mu.Lock() 85 | defer c.mu.Unlock() 86 | h := c.clients[addr] 87 | if h == nil { 88 | h = &clientState{ 89 | addr: addr, 90 | factory: c.factory, 91 | } 92 | c.clients[addr] = h 93 | } 94 | return h 95 | } 96 | 97 | // updateClientState updates h based on its current state and err. 98 | func (c *Client) updateClientState(h *clientState, err error, serviceMethod string) { 99 | c.mu.Lock() 100 | defer c.mu.Unlock() 101 | if err != nil && c.clients[h.addr] == h { 102 | log.Outputf(c.getLogger(h.addr), log.Debug, "resetting http client %s while calling to %s: %s", h.addr, serviceMethod, err.Error()) 103 | delete(c.clients, h.addr) 104 | } 105 | if c.clients[h.addr] != h { 106 | // h is defunct, so we close idle connections to enable collection. 107 | h.cached.CloseIdleConnections() 108 | } 109 | } 110 | 111 | func (c *Client) getLogger(addr string) *rateLimitingOutputter { 112 | v, ok := c.loggers.Load(addr) 113 | if ok { 114 | return v.(*rateLimitingOutputter) 115 | } 116 | v, _ = c.loggers.LoadOrStore(addr, &rateLimitingOutputter{rate.NewLimiter(rate.Every(time.Minute), 1), log.GetOutputter()}) 117 | return v.(*rateLimitingOutputter) 118 | } 119 | 120 | // Call invokes a method on the server named by the provided address. 121 | // The method syntax is "Service.Method": Service is the name of the 122 | // registered service; Method names the method to invoke. 123 | // 124 | // The argument and reply are encoded in accordance with the 125 | // description of the package docs. 126 | // 127 | // If the argument is an io.Reader, it is streamed directly to the 128 | // server method. In this case, Call does not return until the data 129 | // are fully streamed. If the reply is an *io.ReadCloser, the reply 130 | // is streamed directly from the server method. In this case, Call 131 | // returns once the stream is available, and the client is 132 | // responsible for fully reading the data and closing the reader. If 133 | // an error occurs while the response is streamed, the returned 134 | // io.ReadCloser errors on read. 135 | // 136 | // If the argument is a (func () io.Reader) or (func () (io.Reader, error)), it 137 | // is called to get a reader streamed directly to the server method as 138 | // above. This is mostly useful when using Call in a retry loop, as you often 139 | // want to create a new reader for each call, as opposed to continuing from 140 | // whatever unknown state remains from previously attempted calls. 141 | // 142 | // Remote errors are decoded into *errors.Error and returned. 143 | // (Non-*errors.Error errors are converted by the server.) The RPC 144 | // client does not pass on errors of kind errors.Net; these are 145 | // converted to errors.Other. This way, any error of the kind 146 | // errors.Net is guaranteed to originate from the immediate call; 147 | // they are never from the application. 148 | func (c *Client) Call(ctx context.Context, addr, serviceMethod string, arg, reply interface{}) (err error) { 149 | done := clientstats.Start(addr, serviceMethod) 150 | var ( 151 | requestBytes = -1 152 | replyBytes = -1 153 | ) 154 | defer func() { 155 | done(int64(requestBytes), int64(replyBytes), err) 156 | }() 157 | url := strings.TrimRight(addr, "/") + c.prefix + serviceMethod 158 | if log.At(log.Debug) { 159 | call := fmt.Sprint("call ", addr, " ", serviceMethod, " ", truncatef(arg)) 160 | log.Debug.Print(call) 161 | defer func() { 162 | if err != nil { 163 | log.Debug.Print(call, " error: ", err) 164 | } else { 165 | log.Debug.Print(call, " ok: ", truncatef(reply)) 166 | } 167 | }() 168 | } 169 | var ( 170 | body io.Reader 171 | contentType string 172 | ) 173 | switch arg := arg.(type) { 174 | case func() io.Reader: 175 | body = arg() 176 | contentType = "application/octet-stream" 177 | case func() (io.Reader, error): 178 | if body, err = arg(); err != nil { 179 | return errors.E("making reader argument", err) 180 | } 181 | contentType = "application/octet-stream" 182 | case io.Reader: 183 | body = arg 184 | contentType = "application/octet-stream" 185 | default: 186 | b := new(bytes.Buffer) 187 | enc := gob.NewEncoder(b) 188 | if err = enc.Encode(arg); err != nil { 189 | // Because we are writing into a Buffer, any error we see is a 190 | // failure to encode, which will not succeed on retry without 191 | // intervention. 192 | return errors.E(errors.Fatal, errors.Invalid, err) 193 | } 194 | requestBytes = b.Len() 195 | if requestBytes > largeRpcPayload { 196 | log.Outputf(largeArgLogger, log.Info, "call %s %s: large argument: %d bytes", addr, serviceMethod, requestBytes) 197 | } 198 | body = b 199 | contentType = gobContentType 200 | } 201 | 202 | h := c.getClient(addr) 203 | defer func() { 204 | c.updateClientState(h, err, serviceMethod) 205 | }() 206 | resp, err := ctxhttp.Post(ctx, h.Client(), url, contentType, body) 207 | switch err { 208 | case nil: 209 | case context.DeadlineExceeded, context.Canceled: 210 | return err 211 | default: 212 | return errors.E(errors.Net, errors.Temporary, err) 213 | } 214 | if InjectFailures { 215 | resp.Body = &rpcFaultInjector{label: fmt.Sprintf("%s(%s)", serviceMethod, addr), in: resp.Body} 216 | } 217 | switch arg := reply.(type) { 218 | case *io.ReadCloser: 219 | if resp.StatusCode == 200 { 220 | // Wrap the actual response in a stream reader so that errors are 221 | // propagated properly. Callers are responsible for closing the 222 | // stream. 223 | *arg = streamReader{resp} 224 | return nil 225 | } 226 | // In all other cases, we close the body. 227 | defer resp.Body.Close() 228 | switch { 229 | case resp.StatusCode == methodErrorCode: 230 | dec := gob.NewDecoder(resp.Body) 231 | return decodeError(serviceMethod, dec) 232 | case 400 <= resp.StatusCode && resp.StatusCode < 500: 233 | body, err := ioutil.ReadAll(resp.Body) 234 | return errors.E(errors.Fatal, errors.Invalid, fmt.Sprintf("%s: client error %s, %v, %v", url, resp.Status, string(body), err)) 235 | default: 236 | body, err := ioutil.ReadAll(resp.Body) 237 | return errors.E(errors.Fatal, errors.Invalid, fmt.Sprintf("%s: bad reply status %s, %v, %v", url, resp.Status, string(body), err)) 238 | } 239 | default: 240 | defer resp.Body.Close() 241 | sizeReader := &sizeTrackingReader{Reader: resp.Body} 242 | dec := gob.NewDecoder(sizeReader) 243 | switch { 244 | case resp.StatusCode == methodErrorCode: 245 | return decodeError(serviceMethod, dec) 246 | case resp.StatusCode == 200: 247 | err := dec.Decode(reply) 248 | if err != nil { 249 | err = errors.E(errors.Invalid, errors.Temporary, "error while decoding reply for "+serviceMethod, err) 250 | } 251 | replyBytes = sizeReader.Len() 252 | if replyBytes > largeRpcPayload { 253 | log.Outputf(largeReplyLogger, log.Info, "call %s %s: large reply: %d bytes", addr, serviceMethod, replyBytes) 254 | } 255 | return err 256 | case 400 <= resp.StatusCode && resp.StatusCode < 500: 257 | body, err := ioutil.ReadAll(resp.Body) 258 | return errors.E(errors.Fatal, errors.Invalid, fmt.Sprintf("%s: client error %s, %v, %v", url, resp.Status, string(body), err)) 259 | default: 260 | body, err := ioutil.ReadAll(resp.Body) 261 | return errors.E(errors.Fatal, errors.Invalid, fmt.Sprintf("%s: bad reply status %s, %v, %v", url, resp.Status, string(body), err)) 262 | } 263 | } 264 | } 265 | 266 | // StreamReader reads a bigmachine byte stream, propagating 267 | // any errors that may be set in a response's trailer. 268 | type streamReader struct{ *http.Response } 269 | 270 | func (r streamReader) Read(p []byte) (n int, err error) { 271 | n, err = r.Body.Read(p) 272 | if err != io.EOF { 273 | return n, err 274 | } 275 | if e := r.Trailer.Get(bigmachineErrorTrailer); e != "" { 276 | err = errors.New(e) 277 | } 278 | return n, err 279 | } 280 | 281 | func (r streamReader) Close() error { 282 | return r.Body.Close() 283 | } 284 | 285 | func truncatef(v interface{}) string { 286 | b := limitbuf.NewLogger(512) 287 | fmt.Fprint(b, v) 288 | return b.String() 289 | } 290 | 291 | // decodeErrors decodes a serialized error from the codec stream dec. It wraps 292 | // errors with an errors.Remote so that callers can distinguish between errors 293 | // in the machinery to execute the RPC and errors returned by the RPC itself. 294 | func decodeError(serviceMethod string, dec *gob.Decoder) error { 295 | e := new(errors.Error) 296 | if err := dec.Decode(e); err != nil { 297 | return errors.E(errors.Invalid, errors.Temporary, "error while decoding error for "+serviceMethod, err) 298 | } 299 | return errors.E(errors.Remote, e) 300 | } 301 | -------------------------------------------------------------------------------- /rpc/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package rpc implements a simple RPC system for Go methods. Rpc can 6 | // dispatch methods on any Go value of the form: 7 | // 8 | // Func(ctx context.Context, arg argType, reply *replyType) error 9 | // 10 | // By default, the values are Gob-encoded, with the following 11 | // exceptions: 12 | // - if argType is io.Reader, a direct byte stream is provided 13 | // - if replyType is io.ReadCloser, a direct byte stream is provided 14 | // In the future, the package will also support custom codecs, so 15 | // that, for example, if the argument or reply is generated by a 16 | // protocol buffer, then protocol buffer encoding is used 17 | // automatically. 18 | // 19 | // Every value is registered with a name. This name is used by the 20 | // client to specify the object on which to dispatch methods. 21 | // 22 | // Rpc uses HTTP as its transport protocol: the RPC server implements 23 | // an HTTP handler, and exports an HTTP endpoint for each method that 24 | // is served. Similarly, the RPC client composes a HTTP client and 25 | // constructs the appropriate URLs on dispatch. 26 | // 27 | // Each method registered by a server receives its own URL endpoint: 28 | // Service.Method. Calls to a method are performed as HTTP POST 29 | // requests to that method's endpoint. The HTTP body contains a 30 | // gob-encoded (package encoding/gob) stream of data interpreted as 31 | // the method's argument. In the case where the method's argument is 32 | // an io.Reader, the body instead passed through. The reply body 33 | // contains the reply, also gob-encoded, except when the reply has 34 | // type io.ReadCloser in which case the body is passed through and 35 | // streamed end-to-end. 36 | // 37 | // On successful invocation, HTTP code 200 is returned. When a method 38 | // invocation returns an error, HTTP code 590 is returned. In this 39 | // case, the error message is gob-encoded as the reply body. 40 | // 41 | // At the moment, a new gob encoder is created for each call. This is 42 | // inefficient for small requests and replies. Future work includes 43 | // maintaining long-running gob codecs to avoid these inefficiences. 44 | package rpc 45 | 46 | import ( 47 | "bytes" 48 | "context" 49 | "encoding/gob" 50 | "fmt" 51 | "io" 52 | "net/http" 53 | "path" 54 | "reflect" 55 | "runtime/debug" 56 | "strings" 57 | "sync" 58 | 59 | "github.com/grailbio/base/backgroundcontext" 60 | "github.com/grailbio/base/errors" 61 | "github.com/grailbio/base/log" 62 | ) 63 | 64 | // MethodErrorCode is the HTTP error used for method errors. 65 | // On method error, the error message is serialized by the server 66 | // and should be reconstructed by the client. 67 | const methodErrorCode = 590 68 | 69 | // BigmachineErrorTrailer is the HTTP trailer used to 70 | // indicate streaming errors. 71 | const bigmachineErrorTrailer = "x-bigmachine-error" 72 | 73 | var ( 74 | typeOfContext = reflect.TypeOf((*context.Context)(nil)).Elem() 75 | typeOfReader = reflect.TypeOf((*io.Reader)(nil)).Elem() 76 | typeOfReadCloser = reflect.TypeOf((*io.ReadCloser)(nil)).Elem() 77 | typeOfError = reflect.TypeOf((*error)(nil)).Elem() 78 | ) 79 | 80 | // method represents a single method type 81 | type method struct { 82 | method reflect.Method 83 | arg, reply reflect.Type 84 | } 85 | 86 | // A service is a collection of methods invoked on the same receiver value. 87 | type service struct { 88 | name string 89 | recv reflect.Value 90 | typ reflect.Type 91 | methods map[string]*method 92 | } 93 | 94 | // Init initializes a service by inspecting its receiver for 95 | // candidate methods (of the form described in the package docs). 96 | func (s *service) Init() error { 97 | s.methods = make(map[string]*method) 98 | // Search for methods of the form: 99 | // Func(context.Context, argType, *replyType) error 100 | // 101 | // TODO: special cases 102 | // - ProtoMessage() 103 | // - Vanadium structs? 104 | // TODO: provide better ergonomics here, e.g., report methods 105 | // that are almost RPC methods, but don't match one or two of 106 | // the criteria. 107 | for i := 0; i < s.typ.NumMethod(); i++ { 108 | m := s.typ.Method(i) 109 | // Not exported. 110 | if m.PkgPath != "" { 111 | continue 112 | } 113 | // Receiver, context, args, reply 114 | if m.Type.NumIn() != 4 { 115 | continue 116 | } 117 | if m.Type.In(1) != typeOfContext { 118 | continue 119 | } 120 | // TODO: m.Type(2): check that it's exported or builtin 121 | if m.Type.In(3).Kind() != reflect.Ptr { 122 | continue 123 | } 124 | if m.Type.NumOut() != 1 { 125 | continue 126 | } 127 | if m.Type.Out(0) != typeOfError { 128 | continue 129 | } 130 | s.methods[m.Name] = &method{ 131 | method: m, 132 | arg: m.Type.In(2), 133 | reply: m.Type.In(3), 134 | } 135 | } 136 | return nil 137 | } 138 | 139 | // A Server dispatches methods on collection of registered objects. 140 | // Its dispatch rules are described in the package docs. Server 141 | // implements http.Handler and can be served by any HTTP server. 142 | type Server struct { 143 | mu sync.RWMutex 144 | services map[string]*service 145 | } 146 | 147 | // NewServer returns a new, initialized, Server. 148 | func NewServer() *Server { 149 | return &Server{ 150 | services: make(map[string]*service), 151 | } 152 | } 153 | 154 | // Register registers the provided interface under the given name. 155 | // Exported and eligible methods on iface, according to the rules 156 | // described in the package docs, are invoked by this server when 157 | // calls are received from a client. A server dispatches methods 158 | // concurrently. 159 | // 160 | // Register is a noop the a service with the provided name has already been 161 | // registered. 162 | func (s *Server) Register(serviceName string, iface interface{}) error { 163 | s.mu.Lock() 164 | defer s.mu.Unlock() 165 | if s.services[serviceName] != nil { 166 | log.Printf("service %s already defined", serviceName) 167 | return nil 168 | } 169 | svc := &service{ 170 | recv: reflect.ValueOf(iface), 171 | typ: reflect.TypeOf(iface), 172 | name: serviceName, 173 | } 174 | if err := svc.Init(); err != nil { 175 | return err 176 | } 177 | s.services[serviceName] = svc 178 | return nil 179 | } 180 | 181 | // ServeHTTP interprets an HTTP request and, if it represents a valid 182 | // rpc call, dispatches it onto the appropriate registered method. 183 | // 184 | // ServeHTTP implements http.Handler. 185 | func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 186 | if r.Method != "POST" { 187 | http.Error(w, "method not allowed", 405) 188 | return 189 | } 190 | ctx := backgroundcontext.Wrap(r.Context()) 191 | parts := strings.SplitN(path.Base(r.URL.Path), ".", 2) 192 | if len(parts) != 2 { 193 | http.Error(w, "bad url", 400) 194 | return 195 | } 196 | service, method := parts[0], parts[1] 197 | s.mu.RLock() 198 | svc := s.services[service] 199 | s.mu.RUnlock() 200 | if svc == nil { 201 | http.Error(w, "no such service", 404) 202 | return 203 | } 204 | m := svc.methods[method] 205 | if m == nil { 206 | http.Error(w, "no such method", 404) 207 | return 208 | } 209 | defer r.Body.Close() 210 | var ( 211 | err error 212 | requestBytes = -1 213 | replyBytes = -1 214 | ) 215 | done := serverstats.Start("", service+"."+method) 216 | defer func() { 217 | done(int64(requestBytes), int64(replyBytes), err) 218 | }() 219 | // Read the request. 220 | var argv reflect.Value 221 | if m.arg == typeOfReader { 222 | // Readers get the body straight. 223 | argv = reflect.ValueOf(r.Body) 224 | } else { 225 | if m.arg.Kind() == reflect.Ptr { 226 | argv = reflect.New(m.arg.Elem()) 227 | } else { 228 | argv = reflect.New(m.arg) 229 | } 230 | sizeReader := &sizeTrackingReader{Reader: r.Body} 231 | dec := gob.NewDecoder(sizeReader) 232 | requestBytes = sizeReader.Len() 233 | if err = dec.Decode(argv.Interface()); err != nil { 234 | http.Error(w, fmt.Sprintf("error decoding request: %v", err), 400) 235 | return 236 | } 237 | if m.arg.Kind() != reflect.Ptr { 238 | argv = argv.Elem() 239 | } 240 | } 241 | var ( 242 | replyv reflect.Value 243 | readcloser io.ReadCloser 244 | ) 245 | if m.reply.Elem() == typeOfReadCloser { 246 | replyv = reflect.ValueOf(&readcloser) 247 | } else { 248 | replyv = reflect.New(m.reply.Elem()) 249 | switch m.reply.Elem().Kind() { 250 | case reflect.Map: 251 | replyv.Elem().Set(reflect.MakeMap(m.reply.Elem())) 252 | case reflect.Slice: 253 | replyv.Elem().Set(reflect.MakeSlice(m.reply.Elem(), 0, 0)) 254 | } 255 | } 256 | err = func() (err error) { 257 | defer func() { 258 | if e := recover(); e != nil { 259 | log.Error.Printf("panic in method call %s.%s\n%s", service, method, string(debug.Stack())) 260 | err = errors.E(errors.Fatal, fmt.Errorf("panic: %v", e)) 261 | } 262 | }() 263 | rvs := m.method.Func.Call([]reflect.Value{svc.recv, reflect.ValueOf(ctx), argv, replyv}) 264 | if e := rvs[0].Interface(); e != nil { 265 | err = e.(error) 266 | } 267 | return 268 | }() 269 | code := 200 270 | replyIface := replyv.Interface() 271 | if err != nil { 272 | code = methodErrorCode 273 | replyIface = errors.Recover(err) 274 | } 275 | if readcloser != nil { 276 | defer readcloser.Close() 277 | w.Header().Set("Content-Type", "application/octet-stream") 278 | // We pre-declare a trailer so that we can indicate if we encountered an error 279 | // while streaming. 280 | w.Header().Set("Trailer", bigmachineErrorTrailer) 281 | w.WriteHeader(code) 282 | var wr io.Writer = w 283 | if _, needFlush := readcloser.(*flushOpt); needFlush { 284 | if wf, ok := wr.(writeFlusher); ok { 285 | wr = &flusher{wf} 286 | } else { 287 | log.Printf("%s.%s: asked to flush, but HTTP connection does not support flushing", service, method) 288 | } 289 | } 290 | var errStr string 291 | if _, err = io.Copy(wr, readcloser); err != nil { 292 | log.Error.Printf("rpc: error writing reply: %v", err) 293 | errStr = err.Error() 294 | } 295 | // This is required because of a bug in net/http2 that causes the 296 | // connection to hang when pre-declared trailers are not set. 297 | w.Header().Set(bigmachineErrorTrailer, errStr) 298 | return 299 | } 300 | w.Header().Set("Content-Type", gobContentType) 301 | if code != 200 { 302 | // Only write error codes here so that, if the call is a success 303 | // but encoding fails, we have a chance to propagate the error 304 | // properly. 305 | w.WriteHeader(code) 306 | } 307 | b := new(bytes.Buffer) 308 | enc := gob.NewEncoder(b) 309 | err = enc.Encode(replyIface) 310 | replyBytes = b.Len() 311 | if err == nil { 312 | _, err = w.Write(b.Bytes()) 313 | } 314 | if err != nil { 315 | log.Error.Printf("rpc: error writing reply: %v", err) 316 | // May not work, but it's worth a try: 317 | http.Error(w, fmt.Sprint(err), 500) 318 | return 319 | } 320 | } 321 | 322 | // Flush wraps the provided ReadCloser to instruct the rpc server to 323 | // flush after every write. This is useful when the reply stream 324 | // should be interactive -- no guarantees are otherwise provided 325 | // about buffering. 326 | func Flush(rc io.ReadCloser) io.ReadCloser { 327 | return &flushOpt{rc} 328 | } 329 | 330 | type flushOpt struct{ io.ReadCloser } 331 | 332 | type writeFlusher interface { 333 | io.Writer 334 | http.Flusher 335 | } 336 | 337 | type flusher struct{ writeFlusher } 338 | 339 | func (f *flusher) Write(p []byte) (n int, err error) { 340 | n, err = f.writeFlusher.Write(p) 341 | f.writeFlusher.Flush() 342 | return 343 | } 344 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /cmd/multisys/multisys.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | // multisys is an example bigmachine program that uses multiple computing 6 | // clusters to independently estimate π. It illustrates bigmachine's ability to 7 | // use multiple bigmachine.System instances to configure and manage multiple 8 | // computing clusters. 9 | // 10 | // The clusters estimate π using the Monte Carlo method (see bigpi). Each 11 | // cluster collects sample points for the same duration, by default 20s. 12 | // Clusters with more computing power, by way of more or faster machines, will 13 | // consider more sample points. 14 | // 15 | // Clusters are configured as (system, numMachine) pairs. By default, multisys 16 | // uses two clusters using local systems: one with 2 machines and one with 6 17 | // machines. 18 | // 19 | // % multisys 20 | // I0611 22:31:49.682976 45649 multisys.go:226] bigmachine/local:2: status at http://127.0.0.1:3333/bigmachine/0/status 21 | // I0611 22:31:49.682979 45649 multisys.go:226] bigmachine/local:6: status at http://127.0.0.1:3333/bigmachine/1/status 22 | // I0611 22:31:49.774646 45649 multisys.go:234] bigmachine/local:2: waiting for machines to come online 23 | // I0611 22:31:49.775678 45649 multisys.go:234] bigmachine/local:6: waiting for machines to come online 24 | // I0611 22:31:51.919868 45649 multisys.go:237] bigmachine/local:6: machine https://localhost:33139/ RUNNING 25 | // I0611 22:31:52.958645 45649 multisys.go:237] bigmachine/local:6: machine https://localhost:44725/ RUNNING 26 | // I0611 22:31:52.958669 45649 multisys.go:237] bigmachine/local:6: machine https://localhost:37463/ RUNNING 27 | // I0611 22:31:53.005860 45649 multisys.go:237] bigmachine/local:6: machine https://localhost:40629/ RUNNING 28 | // I0611 22:31:53.005878 45649 multisys.go:237] bigmachine/local:6: machine https://localhost:38117/ RUNNING 29 | // I0611 22:31:53.005887 45649 multisys.go:237] bigmachine/local:6: machine https://localhost:37861/ RUNNING 30 | // I0611 22:31:53.005895 45649 multisys.go:242] bigmachine/local:6: all machines are ready 31 | // I0611 22:31:53.005906 45649 multisys.go:267] bigmachine/local:6: distributing work among 6 cores 32 | // I0611 22:31:53.574701 45649 multisys.go:237] bigmachine/local:2: machine https://localhost:46535/ RUNNING 33 | // I0611 22:31:53.574725 45649 multisys.go:237] bigmachine/local:2: machine https://localhost:42687/ RUNNING 34 | // I0611 22:31:53.574738 45649 multisys.go:242] bigmachine/local:2: all machines are ready 35 | // I0611 22:31:53.574759 45649 multisys.go:267] bigmachine/local:2: distributing work among 2 cores 36 | // I0611 22:31:59.024124 45649 local.go:116] machine https://localhost:37861/ terminated 37 | // I0611 22:31:59.024363 45649 local.go:116] machine https://localhost:40629/ terminated 38 | // I0611 22:31:59.024363 45649 local.go:116] machine https://localhost:33139/ terminated 39 | // I0611 22:31:59.024561 45649 local.go:116] machine https://localhost:38117/ terminated 40 | // I0611 22:31:59.024564 45649 local.go:116] machine https://localhost:37463/ terminated 41 | // I0611 22:31:59.024710 45649 local.go:116] machine https://localhost:44725/ terminated 42 | // I0611 22:31:59.583062 45649 local.go:116] machine https://localhost:46535/ terminated 43 | // I0611 22:31:59.583075 45649 local.go:116] machine https://localhost:42687/ terminated 44 | // system machines samples π estimate 45 | // bigmachine/local 2 599000000 3.14169444 46 | // bigmachine/local 6 1799000000 3.141569261 47 | // 48 | // The default is equivalent to running: 49 | // 50 | // % multisys -cluster=bigmachine/local:2 -cluster=bigmachine/local:6 51 | // 52 | // Other clusters can be configured with systems available in the configuration 53 | // profile, loaded from $HOME/grail/profile per grail.Init. 54 | // 55 | // % multisys -cluster=bigmachine/local:3 -cluster=bigmachine/ec2system:2 56 | // I0611 23:40:50.576393 54940 multisys.go:225] bigmachine/ec2system:2: status at http://127.0.0.1:3333/bigmachine/1/status 57 | // I0611 23:40:50.576402 54940 multisys.go:225] bigmachine/local:3: status at http://127.0.0.1:3333/bigmachine/0/status 58 | // I0611 23:40:50.850481 54940 multisys.go:233] bigmachine/local:3: waiting for machines to come online 59 | // I0611 23:40:54.136284 54940 multisys.go:236] bigmachine/local:3: machine https://localhost:39387/ RUNNING 60 | // I0611 23:40:54.319191 54940 multisys.go:236] bigmachine/local:3: machine https://localhost:41691/ RUNNING 61 | // I0611 23:40:54.319213 54940 multisys.go:236] bigmachine/local:3: machine https://localhost:45851/ RUNNING 62 | // I0611 23:40:54.319220 54940 multisys.go:241] bigmachine/local:3: all machines are ready 63 | // I0611 23:40:54.319232 54940 multisys.go:266] bigmachine/local:3: distributing work among 3 cores 64 | // I0611 23:41:00.326318 54940 local.go:116] machine https://localhost:41691/ terminated 65 | // I0611 23:41:00.326343 54940 local.go:116] machine https://localhost:39387/ terminated 66 | // I0611 23:41:00.326537 54940 local.go:116] machine https://localhost:45851/ terminated 67 | // I0611 23:41:07.401206 54940 multisys.go:233] bigmachine/ec2system:2: waiting for machines to come online 68 | // I0611 23:42:19.270927 54940 multisys.go:236] bigmachine/ec2system:2: machine https://ec2-34-215-227-27.us-west-2.compute.amazonaws.com/i-0fa4f9fe0f57aafeb/ RUNNING 69 | // I0611 23:42:19.309629 54940 multisys.go:236] bigmachine/ec2system:2: machine https://ec2-18-237-149-184.us-west-2.compute.amazonaws.com/i-0a8e3d6f8a7d1d3ac/ RUNNING 70 | // I0611 23:42:19.309644 54940 multisys.go:241] bigmachine/ec2system:2: all machines are ready 71 | // I0611 23:42:19.309661 54940 multisys.go:266] bigmachine/ec2system:2: distributing work among 4 cores 72 | // system machines samples π estimate 73 | // bigmachine/local 3 905000000 3.14153730 74 | // bigmachine/ec2system 2 359000000 3.14150672 75 | package main 76 | 77 | import ( 78 | "context" 79 | "encoding/gob" 80 | "flag" 81 | "fmt" 82 | "math" 83 | "math/big" 84 | "math/rand" 85 | "net/http" 86 | _ "net/http/pprof" 87 | "os" 88 | "strconv" 89 | "strings" 90 | "sync" 91 | "sync/atomic" 92 | "text/tabwriter" 93 | "time" 94 | 95 | "github.com/grailbio/base/config" 96 | "github.com/grailbio/base/grail" 97 | "github.com/grailbio/base/log" 98 | "github.com/grailbio/base/must" 99 | "github.com/grailbio/bigmachine" 100 | _ "github.com/grailbio/bigmachine/ec2system" 101 | "golang.org/x/sync/errgroup" 102 | ) 103 | 104 | func init() { 105 | gob.Register(circlePI{}) 106 | } 107 | 108 | type clusterSpec struct { 109 | // instance is the profile instance name of the bigmachine.System used to 110 | // provide machines for the cluster. 111 | instance string 112 | // nMachine is the number of machines in the cluster. 113 | nMachine int 114 | } 115 | 116 | func (s clusterSpec) String() string { 117 | return fmt.Sprintf("%s:%d", s.instance, s.nMachine) 118 | } 119 | 120 | // clusterSpecsValue implements the flag.Value interface and is used to 121 | // configure the different systems/clusters to use to compute π. 122 | type clusterSpecsValue struct { 123 | specs []clusterSpec 124 | } 125 | 126 | func (_ clusterSpecsValue) String() string { 127 | return "" 128 | } 129 | 130 | func (v *clusterSpecsValue) Set(s string) error { 131 | split := strings.Split(s, ":") 132 | if len(split) != 2 { 133 | return fmt.Errorf( 134 | "cluster must be 'system-profile-instance:num-machines', "+ 135 | "e.g. 'bigmachine/ec2system:3': %s", s) 136 | } 137 | nMachine, err := strconv.Atoi(split[1]) 138 | if err != nil { 139 | return fmt.Errorf("num-machines '%s' must be integer: %v", split[1], err) 140 | } 141 | v.specs = append(v.specs, clusterSpec{instance: split[0], nMachine: nMachine}) 142 | return nil 143 | } 144 | 145 | // defaultClusterSpecs defines the clusters to use by default. 146 | var defaultClusterSpecs = []clusterSpec{ 147 | {"bigmachine/local", 2}, 148 | {"bigmachine/local", 6}, 149 | } 150 | 151 | type circlePI struct{} 152 | 153 | // sampleResult holds the result of a sampling. Its fields are exported so that 154 | // they can be gob-encoded. 155 | type sampleResult struct { 156 | // Total is the total number of samples collected. 157 | Total uint64 158 | // In is the number of samples that fell in the unit circle. 159 | In uint64 160 | } 161 | 162 | // Sample generates points inside the unit square for duration d and reports 163 | // how many of these fall inside the unit circle. 164 | func (circlePI) Sample(ctx context.Context, d time.Duration, result *sampleResult) error { 165 | end := time.Now().Add(d) 166 | r := rand.New(rand.NewSource(rand.Int63())) 167 | for time.Now().Before(end) && ctx.Err() == nil { 168 | for i := 0; i < 1e6; i++ { 169 | result.Total++ 170 | x, y := r.Float64(), r.Float64() 171 | if (x-0.5)*(x-0.5)+(y-0.5)*(y-0.5) <= 0.25 { 172 | result.In++ 173 | } 174 | } 175 | } 176 | return ctx.Err() 177 | } 178 | 179 | func main() { 180 | var specsValue clusterSpecsValue 181 | flag.Var( 182 | &specsValue, 183 | "cluster", 184 | "cluster spec as 'profile-instance:num-machines', e.g. 'bigmachine/ec2system:3'", 185 | ) 186 | duration := flag.Duration( 187 | "duration", 188 | 20*time.Second, 189 | "the duration for which clusters will sample points", 190 | ) 191 | grail.Init() 192 | bigmachine.Init() 193 | specs := specsValue.specs 194 | if len(specs) == 0 { 195 | specs = defaultClusterSpecs 196 | } 197 | var ( 198 | ctx = context.Background() 199 | wg sync.WaitGroup 200 | results = make([]sampleResult, len(specs)) 201 | ) 202 | for i, spec := range specs { 203 | i := i 204 | spec := spec 205 | wg.Add(1) 206 | go func() { 207 | results[i] = findPI(ctx, i, spec, *duration) 208 | wg.Done() 209 | }() 210 | } 211 | // Launch a local web server so we have access to profiles. 212 | go func() { 213 | errServe := http.ListenAndServe(":3333", nil) 214 | log.Printf("http.ListenAndServe: %v", errServe) 215 | }() 216 | wg.Wait() 217 | tw := tabwriter.NewWriter(os.Stdout, 0, 4, 4, ' ', tabwriter.AlignRight) 218 | _, err := fmt.Fprintln(tw, "system\tmachines\tsamples\tπ estimate\t") 219 | must.Nil(err) 220 | for i, r := range results { 221 | var ( 222 | pi = big.NewRat(int64(4*r.In), int64(r.Total)) 223 | prec = int(math.Log(float64(r.Total)) / math.Log(10)) 224 | ) 225 | _, err = fmt.Fprintf(tw, "%s\t%d\t%d\t%s\t\n", 226 | specs[i].instance, specs[i].nMachine, r.Total, pi.FloatString(prec)) 227 | must.Nil(err) 228 | } 229 | must.Nil(tw.Flush()) 230 | } 231 | 232 | func findPI(ctx context.Context, idx int, spec clusterSpec, d time.Duration) sampleResult { 233 | var system bigmachine.System 234 | config.Must(spec.instance, &system) 235 | b := bigmachine.Start(system) 236 | defer func() { 237 | b.Shutdown() 238 | // TODO(jcharumilind): Remove this hack once shutdown is synchronous. 239 | time.Sleep(2 * time.Second) 240 | }() 241 | pfx := fmt.Sprintf("/debug/bigmachine/%d/", idx) 242 | b.HandleDebugPrefix(pfx, http.DefaultServeMux) 243 | log.Printf("%s: status at http://127.0.0.1:3333/debug/bigmachine/%d/status", spec, idx) 244 | // Start the desired number of machines, each with the circlePI service. 245 | machines, err := b.Start(ctx, spec.nMachine, bigmachine.Services{ 246 | "PI": circlePI{}, 247 | }) 248 | if err != nil { 249 | log.Fatal(err) 250 | } 251 | log.Printf("%s: waiting for machines to come online", spec) 252 | for _, m := range machines { 253 | <-m.Wait(bigmachine.Running) 254 | log.Printf("%s: machine %s %s", spec, m.Addr, m.State()) 255 | if err := m.Err(); err != nil { 256 | log.Fatal(err) 257 | } 258 | } 259 | log.Printf("%s: all machines are ready", spec) 260 | 261 | // Divide the total number of samples among all the processors on each 262 | // machine. Aggregate the counts and then report the estimate. 263 | var ( 264 | total uint64 265 | in uint64 266 | cores int 267 | ) 268 | g, ctx := errgroup.WithContext(ctx) 269 | for _, m := range machines { 270 | m := m 271 | for i := 0; i < m.Maxprocs; i++ { 272 | cores++ 273 | g.Go(func() error { 274 | var result sampleResult 275 | err := m.Call(ctx, "PI.Sample", d, &result) 276 | if err == nil { 277 | atomic.AddUint64(&total, result.Total) 278 | atomic.AddUint64(&in, result.In) 279 | } 280 | return err 281 | }) 282 | } 283 | } 284 | log.Printf("%s: distributing work among %d cores", spec, cores) 285 | if err := g.Wait(); err != nil { 286 | log.Fatal(err) 287 | } 288 | return sampleResult{Total: total, In: in} 289 | } 290 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bigmachine 2 | 3 | Bigmachine is a toolkit for building self-managing serverless applications 4 | in [Go](https://golang.org/). 5 | Bigmachine provides an API that lets a driver process 6 | form an ad-hoc cluster of machines to 7 | which user code is transparently distributed. 8 | 9 | User code is exposed through services, 10 | which are stateful Go objects associated with each machine. 11 | Services expose one or more Go methods that may 12 | be dispatched remotely. 13 | User services can call remote user services; 14 | the driver process may also make service calls. 15 | 16 | Programs built using Bigmachine are agnostic 17 | to the underlying machine implementation, 18 | allowing distributed systems to be easily tested 19 | through an [in-process implementation](https://godoc.org/github.com/grailbio/bigmachine/testsystem), 20 | or inspected during development using [local Unix processes](https://godoc.org/github.com/grailbio/bigmachine#Local). 21 | 22 | Bigmachine currently supports instantiating clusters of 23 | [EC2 machines](https://godoc.org/github.com/grailbio/bigmachine/ec2system); 24 | other systems may be implemented with a [relatively compact Go interface](https://godoc.org/github.com/grailbio/bigmachine#System). 25 | 26 | - API documentation: [godoc.org/github.com/grailbio/bigmachine](https://godoc.org/github.com/grailbio/bigmachine) 27 | - Issue tracker: [github.com/grailbio/bigmachine/issues](https://github.com/grailbio/bigmachine/issues) 28 | - [![CI](https://github.com/grailbio/bigmachine/workflows/CI/badge.svg)](https://github.com/grailbio/bigmachine/actions?query=workflow%3ACI) 29 | - Implementation notes: [github.com/grailbio/bigmachine/blob/master/docs/impl.md](https://github.com/grailbio/bigmachine/blob/master/docs/impl.md) 30 | 31 | Help wanted! 32 | - [GCP compute engine backend](https://github.com/grailbio/bigmachine/issues/1) 33 | - [Azure VM backend](https://github.com/grailbio/bigmachine/issues/2) 34 | 35 | # A walkthrough of a simple Bigmachine program 36 | 37 | Command [bigpi](https://github.com/grailbio/bigmachine/blob/master/cmd/bigpi/bigpi.go) 38 | is a relatively silly use of cluster computing, 39 | but illustrative nonetheless. 40 | Bigpi estimates the value of $\pi$ 41 | by sampling $N$ random coordinates inside of the unit square, 42 | counting how many $C \le N$ fall inside of the unit circle. 43 | Our estimate is then $\pi = 4*C/N$. 44 | 45 | This is inherently parallelizable: 46 | we can generate samples across a large number of nodes, 47 | and then when we're done, 48 | they can be summed up to produce our estimate of $\pi$. 49 | 50 | To do this in Bigmachine, 51 | we first define a service that samples some $n$ points 52 | and reports how many fell inside the unit circle. 53 | 54 | ``` 55 | type circlePI struct{} 56 | 57 | // Sample generates n points inside the unit square and reports 58 | // how many of these fall inside the unit circle. 59 | func (circlePI) Sample(ctx context.Context, n uint64, m *uint64) error { 60 | r := rand.New(rand.NewSource(rand.Int63())) 61 | for i := uint64(0); i < n; i++ { 62 | if i%1e7 == 0 { 63 | log.Printf("%d/%d", i, n) 64 | } 65 | x, y := r.Float64(), r.Float64() 66 | if (x-0.5)*(x-0.5)+(y-0.5)*(y-0.5) < 0.25 { 67 | *m++ 68 | } 69 | } 70 | return nil 71 | } 72 | ``` 73 | 74 | The only notable aspect of this code is the signature of `Sample`, 75 | which follows the schema below: 76 | methods that follow this convention may be dispatched remotely by Bigmachine, 77 | as we shall see soon. 78 | 79 | ``` 80 | func (service) Name(ctx context.Context, arg argtype, reply *replytype) error 81 | ``` 82 | 83 | Next follows the program's `func main`. 84 | First, we do the regular kind of setup a main might: 85 | define some flags, 86 | parse them, 87 | set up logging. 88 | Afterwards, a driver must call 89 | [`driver.Start`](https://godoc.org/github.com/grailbio/bigmachine/driver#Start), 90 | which initializes Bigmachine 91 | and sets up the process so that it may be bootstrapped properly on remote nodes. 92 | ([Package driver](https://godoc.org/github.com/grailbio/bigmachine/driver) 93 | provides high-level facilities for configuring and bootstrapping Bigmachine; 94 | adventurous users may use the lower-level facilitied in 95 | [package bigmachine](https://godoc.org/github.com/grailbio/bigmachine) 96 | to accomplish the same.) 97 | `driver.Start()` returns a [`*bigmachine.B`](https://godoc.org/gitub.com/grailbio/bigmachine#B) 98 | which can be used to start new machines. 99 | 100 | ``` 101 | func main() { 102 | var ( 103 | nsamples = flag.Int("n", 1e10, "number of samples to make") 104 | nmachine = flag.Int("nmach", 5, "number of machines to provision for the task") 105 | ) 106 | log.AddFlags() 107 | flag.Parse() 108 | b := driver.Start() 109 | defer b.Shutdown() 110 | ``` 111 | 112 | Next, 113 | we start a number of machines (as configured by flag nmach), 114 | wait for them to finish launching, 115 | and then distribute our sampling among them, 116 | using a simple "scatter-gather" RPC pattern. 117 | First, let's look at the code that starts the machines 118 | and waits for them to be ready. 119 | 120 | ``` 121 | // Start the desired number of machines, 122 | // each with the circlePI service. 123 | machines, err := b.Start(ctx, *nmachine, bigmachine.Services{ 124 | "PI": circlePI{}, 125 | }) 126 | if err != nil { 127 | log.Fatal(err) 128 | } 129 | log.Print("waiting for machines to come online") 130 | for _, m := range machines { 131 | <-m.Wait(bigmachine.Running) 132 | log.Printf("machine %s %s", m.Addr, m.State()) 133 | if err := m.Err(); err != nil { 134 | log.Fatal(err) 135 | } 136 | } 137 | log.Print("all machines are ready") 138 | ``` 139 | 140 | Machines are started with [`(*B).Start`](https://godoc.org/github.com/grailbio/bigmachine#B.Start), 141 | to which we provide the set of services that should be installed on each machine. 142 | (The service object provided is serialized and initialized on the remote machine, 143 | so it may include any desired parameters.) 144 | Start returns a slice of 145 | [`Machine`](https://godoc.org/github.com/grailbio/bigmachine#Machine) 146 | instances representing each machine that was launched. 147 | Machines can be in a number of 148 | [states](https://godoc.org/github.com/grailbio/bigmachine#State). 149 | In this case, 150 | we keep it simple and just wait for them to enter their running states, 151 | after which the underlying machines are fully bootstrapped and the services 152 | have been installed and initialized. 153 | At this point, 154 | all of the machines are ready to receive RPC calls. 155 | 156 | The remainder of `main` distributes a portion of 157 | the total samples to be taken to each machine, 158 | waits for them to complete, 159 | and then prints with the precision warranted by the number of samples taken. 160 | Note that this code further subdivides the work by calling PI.Sample 161 | once for each processor available on the underlying machines 162 | as defined by [`Machine.Maxprocs`](https://godoc.org/github.com/grailbio/bigmachine#Machine.Maxprocs), 163 | which depends on the physical machine configuration. 164 | 165 | 166 | ``` 167 | // Number of samples per machine 168 | numPerMachine := uint64(*nsamples) / uint64(*nmachine) 169 | 170 | // Divide the total number of samples among all the processors on 171 | // each machine. Aggregate the counts and then report the estimate. 172 | var total uint64 173 | var cores int 174 | g, ctx := errgroup.WithContext(ctx) 175 | for _, m := range machines { 176 | m := m 177 | for i := 0; i < m.Maxprocs; i++ { 178 | cores++ 179 | g.Go(func() error { 180 | var count uint64 181 | err := m.Call(ctx, "PI.Sample", numPerMachine/uint64(m.Maxprocs), &count) 182 | if err == nil { 183 | atomic.AddUint64(&total, count) 184 | } 185 | return err 186 | }) 187 | } 188 | } 189 | log.Printf("distributing work among %d cores", cores) 190 | if err := g.Wait(); err != nil { 191 | log.Fatal(err) 192 | } 193 | log.Printf("total=%d nsamples=%d", total, *nsamples) 194 | var ( 195 | pi = big.NewRat(int64(4*total), int64(*nsamples)) 196 | prec = int(math.Log(float64(*nsamples)) / math.Log(10)) 197 | ) 198 | fmt.Printf("π = %s\n", pi.FloatString(prec)) 199 | ``` 200 | 201 | We can now build and run our binary like an ordinary Go binary. 202 | 203 | ``` 204 | $ go build 205 | $ ./bigpi 206 | 2019/10/01 16:31:20 waiting for machines to come online 207 | 2019/10/01 16:31:24 machine https://localhost:42409/ RUNNING 208 | 2019/10/01 16:31:24 machine https://localhost:44187/ RUNNING 209 | 2019/10/01 16:31:24 machine https://localhost:41618/ RUNNING 210 | 2019/10/01 16:31:24 machine https://localhost:41134/ RUNNING 211 | 2019/10/01 16:31:24 machine https://localhost:34078/ RUNNING 212 | 2019/10/01 16:31:24 all machines are ready 213 | 2019/10/01 16:31:24 distributing work among 5 cores 214 | 2019/10/01 16:32:05 total=7853881995 nsamples=10000000000 215 | π = 3.1415527980 216 | ``` 217 | 218 | Here, 219 | Bigmachine distributed computation across logical machines, 220 | each corresponding to a single core on the host system. 221 | Each machine ran in its own Unix process (with its own address space), 222 | and RPC happened through mutually authenticated HTTP/2 connections. 223 | 224 | [Package driver](https://godoc.org/github.com/grailbio/bigmachine/driver) 225 | provides some convenient flags that helps configure the Bigmachine runtime. 226 | Using these, we can configure Bigmachine to launch machines into EC2 instead: 227 | 228 | ``` 229 | $ ./bigpi -bigm.system=ec2 230 | 2019/10/01 16:38:10 waiting for machines to come online 231 | 2019/10/01 16:38:43 machine https://ec2-54-244-211-104.us-west-2.compute.amazonaws.com/ RUNNING 232 | 2019/10/01 16:38:43 machine https://ec2-54-189-82-173.us-west-2.compute.amazonaws.com/ RUNNING 233 | 2019/10/01 16:38:43 machine https://ec2-34-221-143-119.us-west-2.compute.amazonaws.com/ RUNNING 234 | ... 235 | 2019/10/01 16:38:43 all machines are ready 236 | 2019/10/01 16:38:43 distributing work among 5 cores 237 | 2019/10/01 16:40:19 total=7853881995 nsamples=10000000000 238 | π = 3.1415527980 239 | ``` 240 | 241 | Once the program is running, 242 | we can use standard Go tooling to examine its behavior. 243 | For example, 244 | [expvars](https://golang.org/pkg/expvar/) 245 | are aggregated across all of the machines managed by Bigmachine, 246 | and the various profiles (CPU, memory, contention, etc.) 247 | are available as merged profiles through `/debug/bigmachine/pprof`. 248 | For example, 249 | in the first version of `bigpi`, 250 | the CPU profile highlighted a problem: 251 | we were using the global `rand.Float64` which requires a lock; 252 | the resulting contention was easily identifiable through the CPU profile: 253 | 254 | ``` 255 | $ go tool pprof localhost:3333/debug/bigmachine/pprof/profile 256 | Fetching profile over HTTP from http://localhost:3333/debug/bigmachine/pprof/profile 257 | Saved profile in /Users/marius/pprof/pprof.045821636.samples.cpu.001.pb.gz 258 | File: 045821636 259 | Type: cpu 260 | Time: Mar 16, 2018 at 3:17pm (PDT) 261 | Duration: 2.51mins, Total samples = 16.80mins (669.32%) 262 | Entering interactive mode (type "help" for commands, "o" for options) 263 | (pprof) top 264 | Showing nodes accounting for 779.47s, 77.31% of 1008.18s total 265 | Dropped 51 nodes (cum <= 5.04s) 266 | Showing top 10 nodes out of 58 267 | flat flat% sum% cum cum% 268 | 333.11s 33.04% 33.04% 333.11s 33.04% runtime.procyield 269 | 116.71s 11.58% 44.62% 469.55s 46.57% runtime.lock 270 | 76.35s 7.57% 52.19% 347.21s 34.44% sync.(*Mutex).Lock 271 | 65.79s 6.53% 58.72% 65.79s 6.53% runtime.futex 272 | 41.48s 4.11% 62.83% 202.05s 20.04% sync.(*Mutex).Unlock 273 | 34.10s 3.38% 66.21% 364.36s 36.14% runtime.findrunnable 274 | 33s 3.27% 69.49% 33s 3.27% runtime.cansemacquire 275 | 32.72s 3.25% 72.73% 51.01s 5.06% runtime.runqgrab 276 | 24.88s 2.47% 75.20% 57.72s 5.73% runtime.unlock 277 | 21.33s 2.12% 77.31% 21.33s 2.12% math/rand.(*rngSource).Uint64 278 | ``` 279 | 280 | And after the fix, 281 | it looks much healthier: 282 | 283 | ``` 284 | $ go tool pprof localhost:3333/debug/bigmachine/pprof/profile 285 | ... 286 | flat flat% sum% cum cum% 287 | 29.09s 35.29% 35.29% 82.43s 100% main.circlePI.Sample 288 | 22.95s 27.84% 63.12% 52.16s 63.27% math/rand.(*Rand).Float64 289 | 16.09s 19.52% 82.64% 16.09s 19.52% math/rand.(*rngSource).Uint64 290 | 9.05s 10.98% 93.62% 25.14s 30.49% math/rand.(*rngSource).Int63 291 | 4.07s 4.94% 98.56% 29.21s 35.43% math/rand.(*Rand).Int63 292 | 1.17s 1.42% 100% 1.17s 1.42% math/rand.New 293 | 0 0% 100% 82.43s 100% github.com/grailbio/bigmachine/rpc.(*Server).ServeHTTP 294 | 0 0% 100% 82.43s 100% github.com/grailbio/bigmachine/rpc.(*Server).ServeHTTP.func2 295 | 0 0% 100% 82.43s 100% golang.org/x/net/http2.(*serverConn).runHandler 296 | 0 0% 100% 82.43s 100% net/http.(*ServeMux).ServeHTTP 297 | ``` 298 | 299 | # GOOS, GOARCH, and Bigmachine 300 | 301 | When using Bigmachine's 302 | [EC2 machine implementation](https://godoc.org/github.com/grailbio/bigmachine/ec2system), 303 | the process is bootstrapped onto remote EC2 instances. 304 | Currently, 305 | the only supported GOOS/GOARCH combination for these are linux/amd64. 306 | Because of this, 307 | the driver program must also be linux/amd64. 308 | However, 309 | Bigmachine also understands the 310 | [fatbin format](https://godoc.org/github.com/grailbio/base/fatbin), 311 | so that users can compile fat binaries using the gofat tool. 312 | For example, 313 | the above can be run on a macOS driver if the binary is built using gofat instead of 'go': 314 | 315 | ``` 316 | macOS $ GO111MODULE=on go get github.com/grailbio/base/cmd/gofat 317 | go: finding github.com/grailbio/base/cmd/gofat latest 318 | go: finding github.com/grailbio/base/cmd latest 319 | macOS $ gofat build 320 | macOS $ ./bigpi -bigm.system=ec2 321 | ... 322 | ``` 323 | -------------------------------------------------------------------------------- /supervisor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 GRAIL, Inc. All rights reserved. 2 | // Use of this source code is governed by the Apache 2.0 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigmachine 6 | 7 | import ( 8 | "context" 9 | "crypto" 10 | "encoding/json" 11 | "expvar" 12 | "fmt" 13 | "io" 14 | "io/ioutil" 15 | "os" 16 | "runtime" 17 | "runtime/pprof" 18 | "strings" 19 | "sync" 20 | "sync/atomic" 21 | "syscall" 22 | "time" 23 | 24 | "github.com/grailbio/base/digest" 25 | "github.com/grailbio/base/errors" 26 | "github.com/grailbio/base/log" 27 | "github.com/grailbio/bigmachine/rpc" 28 | "github.com/shirou/gopsutil/disk" 29 | "github.com/shirou/gopsutil/load" 30 | "github.com/shirou/gopsutil/mem" 31 | ) 32 | 33 | const ( 34 | memProfilePeriod = time.Minute 35 | ) 36 | 37 | var ( 38 | digester = digest.Digester(crypto.SHA256) 39 | binaryDigest digest.Digest 40 | digestOnce sync.Once 41 | ) 42 | 43 | func binary() (io.ReadCloser, error) { 44 | // TODO(marius): use /proc/self/exe on Linux 45 | path, err := os.Executable() 46 | if err != nil { 47 | return nil, err 48 | } 49 | return os.Open(path) 50 | } 51 | 52 | // Supervisor is the system service installed on every machine. 53 | type Supervisor struct { 54 | b *B 55 | system System 56 | server *rpc.Server 57 | nextc chan time.Time 58 | healthy uint32 59 | 60 | mu sync.Mutex 61 | // binaryPath contains the path of the last 62 | // binary uploaded in preparation for Exec. 63 | binaryPath string 64 | environ []string 65 | } 66 | 67 | // StartSupervisor starts a new supervisor based on the provided arguments. 68 | func StartSupervisor(ctx context.Context, b *B, system System, server *rpc.Server) *Supervisor { 69 | s := &Supervisor{ 70 | b: b, 71 | system: system, 72 | server: server, 73 | } 74 | s.healthy = 1 75 | s.nextc = make(chan time.Time) 76 | go s.watchdog(ctx) 77 | return s 78 | } 79 | 80 | // Info contains system information about a machine. 81 | type Info struct { 82 | // Goos and Goarch are the operating system and architectures 83 | // as reported by the Go runtime. 84 | Goos, Goarch string 85 | // Digest is the fingerprint of the currently running binary on the machine. 86 | Digest digest.Digest 87 | // TODO: resources 88 | } 89 | 90 | // LocalInfo returns system information for this process. 91 | func LocalInfo() Info { 92 | digestOnce.Do(func() { 93 | r, err := binary() 94 | if err != nil { 95 | log.Error.Printf("could not read local binary: %v", err) 96 | return 97 | } 98 | defer r.Close() 99 | dw := digester.NewWriter() 100 | if _, err := io.Copy(dw, r); err != nil { 101 | log.Error.Print(err) 102 | return 103 | } 104 | binaryDigest = dw.Digest() 105 | }) 106 | return Info{ 107 | Goos: runtime.GOOS, 108 | Goarch: runtime.GOARCH, 109 | Digest: binaryDigest, 110 | } 111 | } 112 | 113 | type service struct { 114 | Name string 115 | Instance interface{} 116 | } 117 | 118 | // Register registers a new service with the machine (server) associated with 119 | // this supervisor. After registration, the service is also initialized if it implements 120 | // the method 121 | // Init(*B) error 122 | func (s *Supervisor) Register(ctx context.Context, svc service, _ *struct{}) error { 123 | if err := s.server.Register(svc.Name, svc.Instance); err != nil { 124 | return err 125 | } 126 | return maybeInit(svc.Instance, s.b) 127 | } 128 | 129 | // Setargs sets the process' arguments. It should be used before Exec 130 | // in order to invoke the new image with the appropriate arguments. 131 | func (s *Supervisor) Setargs(ctx context.Context, args []string, _ *struct{}) error { 132 | os.Args = args 133 | return nil 134 | } 135 | 136 | // Setenv sets the processes' environment. It is applied to newly exec'd 137 | // images, and should be called before Exec. The provided environment 138 | // is appended to the default process environment: keys provided here 139 | // override those that already exist in the environment. 140 | func (s *Supervisor) Setenv(ctx context.Context, env []string, _ *struct{}) error { 141 | s.mu.Lock() 142 | defer s.mu.Unlock() 143 | s.environ = env 144 | return nil 145 | } 146 | 147 | // Setbinary uploads a new binary to replace the current binary when 148 | // Supervisor.Exec is called. The two calls are separated so that 149 | // different timeouts can be applied to upload and exec. 150 | func (s *Supervisor) Setbinary(ctx context.Context, binary io.Reader, _ *struct{}) error { 151 | f, err := ioutil.TempFile("", "") 152 | if err != nil { 153 | return err 154 | } 155 | if _, err := io.Copy(f, binary); err != nil { 156 | return err 157 | } 158 | path := f.Name() 159 | if err := f.Close(); err != nil { 160 | os.Remove(path) 161 | return err 162 | } 163 | if err := os.Chmod(path, 0755); err != nil { 164 | os.Remove(path) 165 | return err 166 | } 167 | s.mu.Lock() 168 | s.binaryPath = path 169 | s.mu.Unlock() 170 | return nil 171 | } 172 | 173 | // GetBinary retrieves the last binary uploaded via Setbinary. 174 | func (s *Supervisor) GetBinary(ctx context.Context, _ struct{}, rc *io.ReadCloser) error { 175 | s.mu.Lock() 176 | path := s.binaryPath 177 | s.mu.Unlock() 178 | if path == "" { 179 | return errors.E(errors.Invalid, "Supervisor.GetBinary: no binary set") 180 | } 181 | f, err := os.Open(path) 182 | *rc = f 183 | return err 184 | } 185 | 186 | // Exec reads a new image from its argument and replaces the current 187 | // process with it. As a consequence, the currently running machine will 188 | // die. It is up to the caller to manage this interaction. 189 | func (s *Supervisor) Exec(ctx context.Context, _ struct{}, _ *struct{}) error { 190 | s.mu.Lock() 191 | var ( 192 | environ = append(os.Environ(), s.environ...) 193 | path = s.binaryPath 194 | ) 195 | s.mu.Unlock() 196 | if path == "" { 197 | return errors.E(errors.Invalid, "Supervisor.Exec: no binary set") 198 | } 199 | log.Printf("exec %s %s", path, strings.Join(os.Args, " ")) 200 | return syscall.Exec(path, os.Args, environ) 201 | } 202 | 203 | // Ping replies immediately with the sequence number provided. 204 | func (s *Supervisor) Ping(ctx context.Context, seq int, replyseq *int) error { 205 | *replyseq = seq 206 | return nil 207 | } 208 | 209 | // Info returns the info struct for this machine. 210 | func (s *Supervisor) Info(ctx context.Context, _ struct{}, info *Info) error { 211 | *info = LocalInfo() 212 | return nil 213 | } 214 | 215 | // MemInfo returns system and Go runtime memory usage information. 216 | // Go runtime stats are read if readMemStats is true. 217 | func (s *Supervisor) MemInfo(ctx context.Context, readMemStats bool, info *MemInfo) error { 218 | if readMemStats { 219 | runtime.ReadMemStats(&info.Runtime) 220 | } 221 | vm, err := mem.VirtualMemory() 222 | if err != nil { 223 | return err 224 | } 225 | info.System = *vm 226 | return nil 227 | } 228 | 229 | // DiskInfo returns disk usage information on the disk where the 230 | // temporary directory resides. 231 | func (s *Supervisor) DiskInfo(ctx context.Context, _ struct{}, info *DiskInfo) error { 232 | disk, err := disk.Usage(os.TempDir()) 233 | if err != nil { 234 | return err 235 | } 236 | info.Usage = *disk 237 | return nil 238 | } 239 | 240 | // LoadInfo returns system load information. 241 | func (s *Supervisor) LoadInfo(ctx context.Context, _ struct{}, info *LoadInfo) error { 242 | load, err := load.AvgWithContext(ctx) 243 | if err != nil { 244 | return err 245 | } 246 | info.Averages = *load 247 | return nil 248 | } 249 | 250 | // CPUProfile takes a pprof CPU profile of this process for the 251 | // provided duration. If a duration is not provided (is 0) a 252 | // 30-second profile is taken. The profile is returned in the pprof 253 | // serialized form (which uses protocol buffers underneath the hood). 254 | func (s *Supervisor) CPUProfile(ctx context.Context, dur time.Duration, prof *io.ReadCloser) error { 255 | if dur == time.Duration(0) { 256 | dur = 30 * time.Second 257 | } 258 | if !isContextAliveFor(ctx, dur) { 259 | return fmt.Errorf("context is too short for duration %s", dur) 260 | } 261 | r, w := io.Pipe() 262 | *prof = r 263 | go func() { 264 | if err := pprof.StartCPUProfile(w); err != nil { 265 | w.CloseWithError(err) 266 | return 267 | } 268 | var err error 269 | select { 270 | case <-time.After(dur): 271 | case <-ctx.Done(): 272 | err = ctx.Err() 273 | } 274 | pprof.StopCPUProfile() 275 | w.CloseWithError(err) 276 | }() 277 | return nil 278 | } 279 | 280 | type profileRequest struct { 281 | Name string 282 | Debug int 283 | GC bool 284 | } 285 | 286 | // Profile returns the named pprof profile for the current process. 287 | // The profile is returned in protocol buffer format. 288 | func (s *Supervisor) Profile(ctx context.Context, req profileRequest, prof *io.ReadCloser) error { 289 | if req.Name == "heap" && req.GC { 290 | runtime.GC() 291 | } 292 | p := pprof.Lookup(req.Name) 293 | if p == nil { 294 | return fmt.Errorf("no such profile %s", req.Name) 295 | } 296 | r, w := io.Pipe() 297 | *prof = r 298 | go func() { 299 | w.CloseWithError(p.WriteTo(w, req.Debug)) 300 | }() 301 | return nil 302 | } 303 | 304 | type profileStat struct { 305 | Name string 306 | Count int 307 | } 308 | 309 | // Profiles returns the set of available profiles and their counts. 310 | func (s *Supervisor) Profiles(ctx context.Context, _ struct{}, profiles *[]profileStat) error { 311 | for _, p := range pprof.Profiles() { 312 | *profiles = append(*profiles, profileStat{p.Name(), p.Count()}) 313 | } 314 | return nil 315 | } 316 | 317 | // A keepaliveReply stores the reply to a supervisor keepalive request. 318 | type keepaliveReply struct { 319 | // Next is the time until the next expected keepalive. 320 | Next time.Duration 321 | // Healthy indicates whether the supervisor believes the process to 322 | // be healthy. An unhealthy process may soon die. 323 | Healthy bool 324 | } 325 | 326 | // Keepalive maintains the machine keepalive. The next argument 327 | // indicates the callers desired keepalive interval (i.e., the amount 328 | // of time until the keepalive expires from the time of the call); 329 | // the accepted time is returned. In order to maintain the keepalive, 330 | // the driver should call Keepalive again before replynext expires. 331 | func (s *Supervisor) Keepalive(ctx context.Context, next time.Duration, reply *keepaliveReply) error { 332 | now := time.Now() 333 | defer func() { 334 | if diff := time.Since(now); diff > 200*time.Millisecond { 335 | log.Error.Printf("supervisor took a long time to reply to keepalive (%s)", diff) 336 | } 337 | }() 338 | t := now.Add(next) 339 | select { 340 | case s.nextc <- t: 341 | reply.Next = time.Until(t) 342 | reply.Healthy = atomic.LoadUint32(&s.healthy) != 0 343 | return nil 344 | case <-ctx.Done(): 345 | return ctx.Err() 346 | } 347 | } 348 | 349 | // Getpid returns the PID of the supervisor process. 350 | func (s *Supervisor) Getpid(ctx context.Context, _ struct{}, pid *int) error { 351 | *pid = os.Getpid() 352 | return nil 353 | } 354 | 355 | type shutdownRequest struct { 356 | Delay time.Duration 357 | Message string 358 | } 359 | 360 | // Shutdown will cause the process to exit asynchronously at a point 361 | // in the future no sooner than the specified delay. 362 | func (s *Supervisor) Shutdown(ctx context.Context, req shutdownRequest, _ *struct{}) error { 363 | var wg sync.WaitGroup 364 | wg.Add(1) 365 | go func() { 366 | wg.Done() 367 | time.Sleep(req.Delay) 368 | log.Print(req.Message) 369 | s.system.Exit(0) 370 | }() 371 | // Ensure the go routine is scheduled so that the delay is 372 | // more accurate than it otherwise would be. 373 | wg.Wait() 374 | return nil 375 | } 376 | 377 | // An Expvar is a snapshot of an expvar. 378 | type Expvar struct { 379 | Key string 380 | Value string 381 | } 382 | 383 | // Expvars is a collection of snapshotted expvars. 384 | type Expvars []Expvar 385 | 386 | type jsonString string 387 | 388 | func (s jsonString) MarshalJSON() ([]byte, error) { 389 | return []byte(s), nil 390 | } 391 | 392 | func (e Expvars) MarshalJSON() ([]byte, error) { 393 | m := make(map[string]jsonString) 394 | for _, v := range e { 395 | m[v.Key] = jsonString(v.Value) 396 | } 397 | return json.Marshal(m) 398 | } 399 | 400 | // Expvars returns a snapshot of this machine's expvars. 401 | func (s *Supervisor) Expvars(ctx context.Context, _ struct{}, vars *Expvars) error { 402 | expvar.Do(func(kv expvar.KeyValue) { 403 | *vars = append(*vars, Expvar{kv.Key, kv.Value.String()}) 404 | }) 405 | return nil 406 | } 407 | 408 | // TODO(marius): implement a systemd-level watchdog in this routine also. 409 | func (s *Supervisor) watchdog(ctx context.Context) { 410 | var ( 411 | tick = time.NewTicker(30 * time.Second) 412 | // Give a generous initial timeout. 413 | next = time.Now().Add(2 * time.Minute) 414 | lastMemProfile time.Time 415 | ) 416 | for { 417 | select { 418 | case <-tick.C: 419 | case next = <-s.nextc: 420 | case <-ctx.Done(): 421 | return 422 | } 423 | if time.Since(next) > time.Duration(0) { 424 | log.Error.Printf("Watchdog expiration: next=%s", next.Format(time.RFC3339)) 425 | s.system.Exit(1) 426 | } 427 | if time.Since(lastMemProfile) > memProfilePeriod { 428 | vm, err := mem.VirtualMemory() 429 | if err != nil { 430 | // In the case of error, we don't change health status. 431 | log.Error.Printf("failed to retrieve VM stats: %v", err) 432 | continue 433 | } 434 | if used := vm.UsedPercent; used <= 95 { 435 | atomic.StoreUint32(&s.healthy, 1) 436 | } else { 437 | log.Error.Printf("using %.1f%% of system memory; marking machine unhealthy", used) 438 | atomic.StoreUint32(&s.healthy, 0) 439 | } 440 | lastMemProfile = time.Now() 441 | } 442 | } 443 | } 444 | 445 | func isContextAliveFor(ctx context.Context, dur time.Duration) bool { 446 | deadline, ok := ctx.Deadline() 447 | if !ok { 448 | return true 449 | } 450 | return dur <= time.Until(deadline) 451 | } 452 | --------------------------------------------------------------------------------