├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── action.go ├── actions └── actions.go ├── api.go ├── benchmark.go ├── builder └── builder.go ├── cluster.go ├── cmd ├── labagent │ ├── command │ │ └── root.go │ └── main.go ├── labapp │ ├── command │ │ └── root.go │ └── main.go ├── labctl │ ├── command │ │ ├── benchmark.go │ │ ├── build.go │ │ ├── cluster.go │ │ ├── common.go │ │ ├── debug.go │ │ ├── experiment.go │ │ ├── node.go │ │ ├── resolve.go │ │ ├── root.go │ │ └── scenario.go │ └── main.go ├── labd │ ├── command │ │ └── root.go │ └── main.go ├── ociadd │ └── main.go ├── ociget │ └── main.go ├── s3build │ └── main.go ├── s3download │ └── main.go └── s3upload │ └── main.go ├── cue ├── cue.mod │ ├── Makefile │ ├── module.cue │ ├── p2plab.cue │ ├── p2plab_example1.cue │ └── p2plab_example2.cue └── parser │ ├── doc.go │ ├── p2plab_instance.go │ ├── parser.go │ └── parser_test.go ├── daemon ├── cancel_closer.go ├── daemon.go ├── healthcheckrouter │ └── router.go └── router.go ├── dag └── walker.go ├── distribution.go ├── downloaders ├── downloaders.go ├── filedownloader │ └── downloader.go ├── httpdownloader │ └── downloader.go └── s3downloader │ └── downloader.go ├── errdefs └── errors.go ├── examples ├── cluster │ ├── multiple-regions.json │ ├── quic.json │ └── same-region.json ├── peer │ └── default.json └── scenario │ ├── cross-region.json │ ├── image-deduplication.json │ └── neighbors.json ├── experiment.go ├── experiments ├── definition.go ├── experiments_test.go ├── report.go └── report_test.go ├── go.mod ├── go.sum ├── labagent ├── agentapi │ └── agentapi.go ├── agentrouter │ └── router.go ├── labagent.go ├── settings.go └── supervisor │ └── supervisor.go ├── labapp ├── appapi │ └── appapi.go ├── approuter │ └── router.go └── labapp.go ├── labd ├── controlapi │ ├── benchmark.go │ ├── build.go │ ├── cluster.go │ ├── controlapi.go │ ├── experiment.go │ ├── node.go │ └── scenario.go ├── labd.go ├── routers │ ├── benchmarkrouter │ │ └── router.go │ ├── buildrouter │ │ └── router.go │ ├── clusterrouter │ │ └── router.go │ ├── experimentrouter │ │ └── router.go │ ├── helpers │ │ ├── doc.go │ │ └── helpers.go │ ├── noderouter │ │ └── router.go │ └── scenariorouter │ │ └── router.go └── settings.go ├── metadata ├── benchmark.go ├── buckets.go ├── build.go ├── cluster.go ├── db.go ├── db_test.go ├── experiment.go ├── helpers.go ├── node.go ├── report.go ├── scenario.go ├── trial.go └── validate.go ├── node.go ├── nodes ├── connect.go ├── healthy.go ├── reports.go ├── session.go └── update.go ├── peer.go ├── peer ├── libp2p.go └── peer.go ├── pkg ├── cliutil │ ├── attach.go │ └── interrupt.go ├── digestconv │ ├── digestconv.go │ └── digestconv_test.go ├── httputil │ ├── client.go │ ├── handler.go │ └── request.go ├── logutil │ ├── console.go │ ├── elapsed.go │ ├── http.go │ ├── jaeger.go │ ├── remote.go │ └── writer.go ├── stringutil │ └── stringutil.go └── traceutil │ └── tracer.go ├── printer ├── id.go ├── json.go ├── printer.go ├── report.go ├── table.go └── unix.go ├── providers ├── inmemory │ └── provider.go ├── providers.go └── terraform │ ├── instances.go │ ├── provider.go │ ├── templates │ ├── main.tf │ ├── modules │ │ └── labagent │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ ├── outputs.tf │ ├── terraform.tfvars │ └── variables.tf │ └── terraform.go ├── query.go ├── query ├── execute.go ├── execute_test.go ├── labeled.go ├── labeled_set.go ├── labeled_set_test.go └── query.go ├── reports └── aggregates.go ├── scenario.go ├── scenarios ├── definition.go ├── plan.go └── run.go ├── transformer.go ├── transformers ├── oci │ ├── content.go │ ├── metadata.go │ └── oci.go └── transformers.go ├── uploaders ├── fileuploader │ └── uploader.go ├── s3uploader │ └── uploader.go └── uploaders.go └── version └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | tmp/ 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | os: 4 | - linux 5 | - osx 6 | 7 | env: 8 | global: 9 | - GO111MODULE=on 10 | 11 | go: 12 | - "1.14.1" 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all binaries test clean 2 | 3 | all: binaries test 4 | 5 | binaries: cmd/labd cmd/labctl cmd/labagent cmd/labapp 6 | 7 | FORCE: 8 | 9 | cmd/%: FORCE 10 | @echo "$@" 11 | @go build -o "./bin/$$(basename $@)" "./$@" 12 | 13 | test: 14 | @echo "$@" 15 | @go test -v ./... 16 | 17 | clean: 18 | @echo "$@" 19 | @rm -rf ./bin ./tmp 20 | -------------------------------------------------------------------------------- /action.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | type Action interface { 24 | String() string 25 | 26 | Tasks(ctx context.Context, ns []Node) (map[string]metadata.Task, error) 27 | } 28 | -------------------------------------------------------------------------------- /actions/actions.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package actions 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/Netflix/p2plab" 22 | "github.com/Netflix/p2plab/metadata" 23 | cid "github.com/ipfs/go-cid" 24 | ) 25 | 26 | func Parse(objects map[string]cid.Cid, a string) (p2plab.Action, error) { 27 | return &dummyAction{objects[a].String()}, nil 28 | } 29 | 30 | type dummyAction struct { 31 | subject string 32 | } 33 | 34 | func (a *dummyAction) String() string { 35 | return fmt.Sprintf("get %q", a.subject) 36 | } 37 | 38 | func (a *dummyAction) Tasks(ctx context.Context, ns []p2plab.Node) (map[string]metadata.Task, error) { 39 | taskMap := make(map[string]metadata.Task) 40 | for _, n := range ns { 41 | taskMap[n.Metadata().ID] = metadata.Task{ 42 | Type: metadata.TaskGet, 43 | Subject: a.subject, 44 | } 45 | } 46 | return taskMap, nil 47 | } 48 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | "github.com/libp2p/go-libp2p-core/peer" 22 | ) 23 | 24 | // ControlAPI defines APIs for labd. 25 | type ControlAPI interface { 26 | // Cluster returns an implementaiton of Cluster API. 27 | Cluster() ClusterAPI 28 | 29 | // Node returns an implementation of Node API. 30 | Node() NodeAPI 31 | 32 | // Scenario returns an implementation of Scenario API. 33 | Scenario() ScenarioAPI 34 | 35 | // Benchmark returns an implementation of Benchmark API. 36 | Benchmark() BenchmarkAPI 37 | 38 | // Experiment returns an implementation of Experiment API. 39 | Experiment() ExperimentAPI 40 | 41 | // Build returns an implementation of Build API. 42 | Build() BuildAPI 43 | } 44 | 45 | type AgentAPI interface { 46 | Healthcheck(ctx context.Context) bool 47 | 48 | Update(ctx context.Context, id, link string, pdef metadata.PeerDefinition) error 49 | 50 | // SSH creates a SSH connection to the node. 51 | SSH(ctx context.Context, opts ...SSHOption) error 52 | } 53 | 54 | type AppAPI interface { 55 | PeerInfo(ctx context.Context) (peer.AddrInfo, error) 56 | 57 | Report(ctx context.Context) (metadata.ReportNode, error) 58 | 59 | // Run executes an task on the node. 60 | Run(ctx context.Context, task metadata.Task) error 61 | } 62 | -------------------------------------------------------------------------------- /benchmark.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | // BenchmarkAPI defines API for benchmark operations. 24 | type BenchmarkAPI interface { 25 | // Create creates a benchmark of a scenario on a cluster. 26 | Create(ctx context.Context, cluster, scenario string, opts ...StartBenchmarkOption) (id string, err error) 27 | 28 | // Get returns a benchmark. 29 | Get(ctx context.Context, id string) (Benchmark, error) 30 | 31 | Label(ctx context.Context, ids, adds, removes []string) ([]Benchmark, error) 32 | 33 | // List returns available benchmarks. 34 | List(ctx context.Context, opts ...ListOption) ([]Benchmark, error) 35 | 36 | Remove(ctx context.Context, ids ...string) error 37 | } 38 | 39 | // Benchmark is an execution of a scenario on a cluster. 40 | type Benchmark interface { 41 | Labeled 42 | 43 | Metadata() metadata.Benchmark 44 | 45 | Report(ctx context.Context) (metadata.Report, error) 46 | } 47 | 48 | type StartBenchmarkOption func(*StartBenchmarkSettings) error 49 | 50 | type StartBenchmarkSettings struct { 51 | NoReset bool 52 | } 53 | 54 | func WithBenchmarkNoReset() StartBenchmarkOption { 55 | return func(s *StartBenchmarkSettings) error { 56 | s.NoReset = true 57 | return nil 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | // ClusterAPI defines API for cluster operations. 24 | type ClusterAPI interface { 25 | // Create deploys a cluster. 26 | Create(ctx context.Context, name string, opts ...CreateClusterOption) (id string, err error) 27 | 28 | // Get returns a cluster. 29 | Get(ctx context.Context, name string) (Cluster, error) 30 | 31 | // Label adds/removes labels to/from clusters. 32 | Label(ctx context.Context, names, adds, removes []string) ([]Cluster, error) 33 | 34 | // List returns available clusters. 35 | List(ctx context.Context, opts ...ListOption) ([]Cluster, error) 36 | 37 | // Remove destroys clusters permanently. 38 | Remove(ctx context.Context, names ...string) error 39 | } 40 | 41 | // Cluster is a group of instances connected in a p2p network. They can be 42 | // provisioned by developers, or CI. Clusters may span multiple regions and 43 | // have heterogeneous nodes. 44 | type Cluster interface { 45 | Labeled 46 | 47 | Metadata() metadata.Cluster 48 | 49 | // Update updates nodes in a cluster matching the list options with the given 50 | // peer definition. 51 | Update(ctx context.Context, pdef metadata.PeerDefinition, opts ...ListOption) ([]Node, error) 52 | } 53 | 54 | // CreateClusterOption is an option to modify create cluster settings. 55 | type CreateClusterOption func(*CreateClusterSettings) error 56 | 57 | // CreateClusterSettings specify cluster properties for creation. 58 | type CreateClusterSettings struct { 59 | Definition string 60 | Size int 61 | InstanceType string 62 | Region string 63 | ClusterDefinition metadata.ClusterDefinition 64 | } 65 | 66 | func WithClusterDefinition(definition string) CreateClusterOption { 67 | return func(s *CreateClusterSettings) error { 68 | s.Definition = definition 69 | return nil 70 | } 71 | } 72 | 73 | func WithClusterSize(size int) CreateClusterOption { 74 | return func(s *CreateClusterSettings) error { 75 | s.Size = size 76 | return nil 77 | } 78 | } 79 | 80 | func WithClusterInstanceType(instanceType string) CreateClusterOption { 81 | return func(s *CreateClusterSettings) error { 82 | s.InstanceType = instanceType 83 | return nil 84 | } 85 | } 86 | 87 | func WithClusterRegion(region string) CreateClusterOption { 88 | return func(s *CreateClusterSettings) error { 89 | s.Region = region 90 | return nil 91 | } 92 | } 93 | 94 | type ListOption func(*ListSettings) error 95 | 96 | type ListSettings struct { 97 | Query string 98 | } 99 | 100 | func WithQuery(q string) ListOption { 101 | return func(s *ListSettings) error { 102 | s.Query = q 103 | return nil 104 | } 105 | } 106 | 107 | type QueryOption func(*QuerySettings) error 108 | 109 | type QuerySettings struct { 110 | AddLabels []string 111 | RemoveLabels []string 112 | } 113 | 114 | func WithAddLabels(labels ...string) QueryOption { 115 | return func(s *QuerySettings) error { 116 | s.AddLabels = append(s.AddLabels, labels...) 117 | return nil 118 | } 119 | } 120 | 121 | func WithRemoveLabels(labels ...string) QueryOption { 122 | return func(s *QuerySettings) error { 123 | s.RemoveLabels = append(s.RemoveLabels, labels...) 124 | return nil 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /cmd/labagent/command/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package command 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/Netflix/p2plab/downloaders" 23 | "github.com/Netflix/p2plab/downloaders/s3downloader" 24 | "github.com/Netflix/p2plab/labagent" 25 | "github.com/Netflix/p2plab/pkg/cliutil" 26 | "github.com/Netflix/p2plab/providers/terraform" 27 | "github.com/Netflix/p2plab/version" 28 | "github.com/rs/zerolog" 29 | "github.com/urfave/cli" 30 | ) 31 | 32 | func App(ctx context.Context) *cli.App { 33 | app := cli.NewApp() 34 | app.Name = "labagent" 35 | app.Version = version.Version 36 | app.Flags = []cli.Flag{ 37 | cli.StringFlag{ 38 | Name: "root", 39 | Usage: "path to state directory", 40 | Value: "./tmp/labagent", 41 | EnvVar: "LABAGENT_ROOT", 42 | }, 43 | cli.StringFlag{ 44 | Name: "address,a", 45 | Usage: "address for labd's HTTP server", 46 | Value: fmt.Sprintf(":%d", terraform.DefaultAgentPort), 47 | EnvVar: "LABAGENT_ADDRESS", 48 | }, 49 | cli.StringFlag{ 50 | Name: "app-root", 51 | Usage: "path to labapp's state directory", 52 | Value: "./tmp/labapp", 53 | EnvVar: "LABAGENT_APP_ROOT", 54 | }, 55 | cli.StringFlag{ 56 | Name: "app-address", 57 | Usage: "address for labapp's HTTP server", 58 | Value: fmt.Sprintf("http://localhost:%d", terraform.DefaultAppPort), 59 | EnvVar: "LABAGENT_APP_ADDRESS", 60 | }, 61 | cli.StringFlag{ 62 | Name: "log-level,l", 63 | Usage: "set the logging level [debug, info, warn, error, fatal, panic]", 64 | Value: "debug", 65 | EnvVar: "LABAGENT_LOG_LEVEL", 66 | }, 67 | cli.StringFlag{ 68 | Name: "downloader.s3.region", 69 | Usage: "region for s3 downloader", 70 | EnvVar: "LABAGENT_DOWNLOADER_S3_REGION", 71 | }, 72 | } 73 | app.Action = agentAction 74 | 75 | // Setup context. 76 | cliutil.AttachAppContext(ctx, app) 77 | 78 | return app 79 | } 80 | 81 | func agentAction(c *cli.Context) error { 82 | root := c.GlobalString("root") 83 | err := os.MkdirAll(root, 0711) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | ctx := cliutil.CommandContext(c) 89 | agent, err := labagent.New(root, c.String("address"), c.String("app-root"), c.String("app-address"), zerolog.Ctx(ctx), 90 | labagent.WithDownloaderSettings(downloaders.DownloaderSettings{ 91 | S3: s3downloader.S3DownloaderSettings{ 92 | Region: c.String("downloader.s3.region"), 93 | }, 94 | }), 95 | ) 96 | if err != nil { 97 | return err 98 | } 99 | defer agent.Close() 100 | 101 | return agent.Serve(ctx) 102 | } 103 | -------------------------------------------------------------------------------- /cmd/labagent/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "syscall" 22 | 23 | "github.com/Netflix/p2plab/cmd/labagent/command" 24 | "github.com/Netflix/p2plab/pkg/cliutil" 25 | "github.com/rs/zerolog" 26 | ) 27 | 28 | func init() { 29 | // UNIX Time is faster and smaller than most timestamps. If you set 30 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 31 | // time. 32 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs 33 | } 34 | 35 | func main() { 36 | ctx, cancel := context.WithCancel(context.Background()) 37 | 38 | ih := cliutil.NewInterruptHandler(cancel, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) 39 | defer ih.Close() 40 | 41 | app := command.App(ctx) 42 | if err := app.Run(os.Args); err != nil { 43 | fmt.Fprintf(os.Stderr, "labagent: %s\n", err) 44 | os.Exit(1) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /cmd/labapp/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "syscall" 22 | 23 | "github.com/Netflix/p2plab/cmd/labapp/command" 24 | "github.com/Netflix/p2plab/pkg/cliutil" 25 | "github.com/rs/zerolog" 26 | ) 27 | 28 | func init() { 29 | // UNIX Time is faster and smaller than most timestamps. If you set 30 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 31 | // time. 32 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs 33 | } 34 | 35 | func main() { 36 | ctx, cancel := context.WithCancel(context.Background()) 37 | 38 | ih := cliutil.NewInterruptHandler(cancel, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) 39 | defer ih.Close() 40 | 41 | app := command.App(ctx) 42 | if err := app.Run(os.Args); err != nil { 43 | fmt.Fprintf(os.Stderr, "labapp: %s\n", err) 44 | os.Exit(1) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /cmd/labctl/command/resolve.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package command 16 | 17 | import ( 18 | "github.com/Netflix/p2plab" 19 | "github.com/Netflix/p2plab/labagent/agentapi" 20 | "github.com/Netflix/p2plab/labapp/appapi" 21 | "github.com/Netflix/p2plab/labd/controlapi" 22 | "github.com/urfave/cli" 23 | ) 24 | 25 | func ResolveControl(c *cli.Context) (p2plab.ControlAPI, error) { 26 | api := controlapi.New(CommandClient(c), c.GlobalString("address")) 27 | // TODO: healthcheck 28 | return api, nil 29 | } 30 | 31 | func ResolveAgent(c *cli.Context, addr string) (p2plab.AgentAPI, error) { 32 | api := agentapi.New(CommandClient(c), addr) 33 | // TODO: healthcheck 34 | return api, nil 35 | } 36 | 37 | func ResolveApp(c *cli.Context, addr string) (p2plab.AppAPI, error) { 38 | api := appapi.New(CommandClient(c), addr) 39 | // TODO: healthcheck 40 | return api, nil 41 | } 42 | -------------------------------------------------------------------------------- /cmd/labctl/command/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package command 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/version" 21 | "github.com/urfave/cli" 22 | ) 23 | 24 | func App(ctx context.Context) *cli.App { 25 | app := cli.NewApp() 26 | app.Name = "labctl" 27 | app.Version = version.Version 28 | app.Flags = []cli.Flag{ 29 | cli.StringFlag{ 30 | Name: "address,a", 31 | Usage: "address for labd", 32 | Value: "http://127.0.0.1:7001", 33 | EnvVar: "LABCTL_ADDRESS", 34 | }, 35 | cli.StringFlag{ 36 | Name: "log-level", 37 | Usage: "set the logging level [debug, info, warn, error, fatal, panic]", 38 | Value: "info", 39 | EnvVar: "LABCTL_LOG_LEVEL", 40 | }, 41 | cli.StringFlag{ 42 | Name: "log-writer", 43 | Usage: "set the log writer [console, json]", 44 | Value: "console", 45 | EnvVar: "LABCTL_LOG_WRITER", 46 | }, 47 | cli.StringFlag{ 48 | Name: "output,o", 49 | Usage: "set the output printer [auto, id, unix, json, table]", 50 | Value: "auto", 51 | EnvVar: "LABCTL_OUTPUT", 52 | }, 53 | } 54 | app.Commands = []cli.Command{ 55 | clusterCommand, 56 | nodeCommand, 57 | scenarioCommand, 58 | benchmarkCommand, 59 | experimentCommand, 60 | buildCommand, 61 | debugCommand, 62 | } 63 | 64 | // Setup tracers and context. 65 | AttachAppContext(ctx, app) 66 | 67 | // Setup http client. 68 | AttachAppClient(app) 69 | 70 | return app 71 | } 72 | -------------------------------------------------------------------------------- /cmd/labctl/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "syscall" 22 | 23 | "github.com/Netflix/p2plab/cmd/labctl/command" 24 | "github.com/Netflix/p2plab/pkg/cliutil" 25 | "github.com/rs/zerolog" 26 | ) 27 | 28 | func init() { 29 | // UNIX Time is faster and smaller than most timestamps. If you set 30 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 31 | // time. 32 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs 33 | } 34 | 35 | func main() { 36 | ctx, cancel := context.WithCancel(context.Background()) 37 | 38 | ih := cliutil.NewInterruptHandler(cancel, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) 39 | defer ih.Close() 40 | 41 | app := command.App(ctx) 42 | if err := app.Run(os.Args); err != nil { 43 | fmt.Fprintf(os.Stderr, "labctl: %s\n", err) 44 | os.Exit(1) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /cmd/labd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "syscall" 22 | 23 | "github.com/Netflix/p2plab/cmd/labd/command" 24 | "github.com/Netflix/p2plab/pkg/cliutil" 25 | "github.com/rs/zerolog" 26 | ) 27 | 28 | func init() { 29 | // UNIX Time is faster and smaller than most timestamps. If you set 30 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 31 | // time. 32 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs 33 | } 34 | 35 | func main() { 36 | ctx, cancel := context.WithCancel(context.Background()) 37 | 38 | ih := cliutil.NewInterruptHandler(cancel, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) 39 | defer ih.Close() 40 | 41 | app := command.App(ctx) 42 | if err := app.Run(os.Args); err != nil { 43 | fmt.Fprintf(os.Stderr, "labd: %s\n", err) 44 | os.Exit(1) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /cmd/ociadd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bufio" 19 | "context" 20 | "fmt" 21 | "os" 22 | "path/filepath" 23 | 24 | "github.com/Netflix/p2plab/metadata" 25 | "github.com/Netflix/p2plab/peer" 26 | "github.com/Netflix/p2plab/pkg/httputil" 27 | "github.com/Netflix/p2plab/transformers/oci" 28 | "github.com/rs/zerolog" 29 | "github.com/rs/zerolog/log" 30 | ) 31 | 32 | func init() { 33 | // UNIX Time is faster and smaller than most timestamps. If you set 34 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 35 | // time. 36 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 37 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 38 | } 39 | 40 | func main() { 41 | if len(os.Args) != 2 { 42 | fmt.Fprintf(os.Stderr, "ociadd: must specify ref") 43 | os.Exit(1) 44 | } 45 | 46 | err := run(os.Args[1]) 47 | if err != nil { 48 | fmt.Fprintf(os.Stderr, "ociadd: %s\n", err) 49 | os.Exit(1) 50 | } 51 | } 52 | 53 | func run(ref string) error { 54 | ctx, cancel := context.WithCancel(context.Background()) 55 | defer cancel() 56 | 57 | root := "./tmp/ociadd" 58 | err := os.MkdirAll(root, 0711) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | p, err := peer.New(ctx, filepath.Join(root, "peer"), 0, metadata.PeerDefinition{ 64 | Transports: []string{"quic"}, 65 | Muxers: []string{"mplex"}, 66 | SecurityTransports: []string{"secio"}, 67 | Routing: "nil", 68 | }) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | var addrs []string 74 | for _, ma := range p.Host().Addrs() { 75 | addrs = append(addrs, ma.String()) 76 | } 77 | log.Info().Str("id", p.Host().ID().String()).Strs("listen", addrs).Msg("Starting libp2p peer") 78 | 79 | transformer, err := oci.New(filepath.Join(root, "transformers/oci"), httputil.NewHTTPClient()) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | c, err := transformer.Transform(ctx, p, ref) 85 | if err != nil { 86 | return err 87 | } 88 | log.Info().Str("ref", ref).Str("cid", c.String()).Msg("Converted OCI image to IPLD DAG") 89 | 90 | log.Info().Msgf("Retrieve manifest from another p2plab/peer by running:\n\ngo run ./cmd/ociget %s/p2p/%s %s\n", p.Host().Addrs()[0], p.Host().ID(), c) 91 | 92 | log.Info().Msgf("Connect to this peer from IPFS daemon:\n\nipfs swarm connect %s/p2p/%s\nipfs cat %s\n", p.Host().Addrs()[0], p.Host().ID(), c) 93 | 94 | fmt.Print("Press 'Enter' to terminate peer...") 95 | _, err = bufio.NewReader(os.Stdin).ReadBytes('\n') 96 | if err != nil { 97 | return err 98 | } 99 | 100 | return nil 101 | } 102 | -------------------------------------------------------------------------------- /cmd/ociget/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bufio" 19 | "context" 20 | "fmt" 21 | "os" 22 | "path/filepath" 23 | 24 | "github.com/Netflix/p2plab/metadata" 25 | "github.com/Netflix/p2plab/peer" 26 | cid "github.com/ipfs/go-cid" 27 | files "github.com/ipfs/go-ipfs-files" 28 | libp2ppeer "github.com/libp2p/go-libp2p-core/peer" 29 | multiaddr "github.com/multiformats/go-multiaddr" 30 | "github.com/pkg/errors" 31 | "github.com/rs/zerolog" 32 | "github.com/rs/zerolog/log" 33 | ) 34 | 35 | func init() { 36 | // UNIX Time is faster and smaller than most timestamps. If you set 37 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 38 | // time. 39 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 40 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 41 | } 42 | 43 | func main() { 44 | if len(os.Args) != 3 { 45 | fmt.Fprintf(os.Stderr, "ociget: must specify peer addr and cid") 46 | os.Exit(1) 47 | } 48 | 49 | err := run(os.Args[1], os.Args[2]) 50 | if err != nil { 51 | fmt.Fprintf(os.Stderr, "ociget: %s\n", err) 52 | os.Exit(1) 53 | } 54 | } 55 | 56 | func run(addr, ref string) error { 57 | ctx, cancel := context.WithCancel(context.Background()) 58 | defer cancel() 59 | 60 | root := "./tmp/ociget" 61 | err := os.MkdirAll(root, 0711) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | p, err := peer.New(ctx, filepath.Join(root, "peer"), 0, metadata.PeerDefinition{ 67 | Transports: []string{"tcp", "ws", "quic"}, 68 | Muxers: []string{"mplex", "yamux"}, 69 | SecurityTransports: []string{"tls", "secio"}, 70 | Routing: "nil", 71 | }) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | var addrs []string 77 | for _, ma := range p.Host().Addrs() { 78 | addrs = append(addrs, ma.String()) 79 | } 80 | log.Info().Str("id", p.Host().ID().String()).Strs("listen", addrs).Msg("Starting libp2p peer") 81 | 82 | targetAddr, err := multiaddr.NewMultiaddr(addr) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | targetInfo, err := libp2ppeer.AddrInfoFromP2pAddr(targetAddr) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | err = p.Host().Connect(ctx, *targetInfo) 93 | if err != nil { 94 | return err 95 | } 96 | log.Info().Str("addr", targetAddr.String()).Msg("Connected to peer") 97 | 98 | c, err := cid.Parse(ref) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | nd, err := p.Get(ctx, c) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | f, ok := nd.(files.Directory) 109 | if !ok { 110 | return errors.Errorf("expected %q to be a directory", c) 111 | } 112 | 113 | iter := f.Entries() 114 | for iter.Next() { 115 | log.Info().Str("name", iter.Name()).Msg("Found file in unixfs directory") 116 | } 117 | 118 | if iter.Err() != nil { 119 | return err 120 | } 121 | 122 | fmt.Print("Press 'Enter' to terminate peer...") 123 | _, err = bufio.NewReader(os.Stdin).ReadBytes('\n') 124 | if err != nil { 125 | return err 126 | } 127 | 128 | return nil 129 | } 130 | -------------------------------------------------------------------------------- /cmd/s3build/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | "os" 22 | 23 | "github.com/Netflix/p2plab/builder" 24 | "github.com/Netflix/p2plab/metadata" 25 | "github.com/Netflix/p2plab/pkg/httputil" 26 | "github.com/Netflix/p2plab/uploaders/s3uploader" 27 | "github.com/rs/zerolog" 28 | "github.com/rs/zerolog/log" 29 | ) 30 | 31 | func init() { 32 | // UNIX Time is faster and smaller than most timestamps. If you set 33 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 34 | // time. 35 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 36 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 37 | } 38 | 39 | func main() { 40 | if len(os.Args) != 2 { 41 | fmt.Fprintf(os.Stderr, "s3build: must specify commit to build") 42 | os.Exit(1) 43 | } 44 | 45 | err := run(os.Args[1]) 46 | if err != nil { 47 | fmt.Fprintf(os.Stderr, "s3build: %s\n", err) 48 | os.Exit(1) 49 | } 50 | } 51 | 52 | func run(ref string) error { 53 | logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger() 54 | ctx := logger.WithContext(context.Background()) 55 | 56 | root := "./tmp/s3build" 57 | err := os.MkdirAll(root, 0711) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | db, err := metadata.NewDB(root) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | client := httputil.NewHTTPClient() 68 | uploader, err := s3uploader.New(client, s3uploader.S3UploaderSettings{ 69 | Bucket: os.Getenv("S3_UPLOADER_BUCKET"), 70 | Prefix: os.Getenv("S3_UPLOADER_PREFIX"), 71 | }) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | builder, err := builder.New(root, db, uploader) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | err = builder.Init(ctx) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | commit, err := builder.Resolve(ctx, ref) 87 | if err != nil { 88 | return err 89 | } 90 | zerolog.Ctx(ctx).Info().Str("ref", ref).Str("commit", commit).Msg("Resolved ref") 91 | 92 | link, err := builder.Build(ctx, commit) 93 | if err != nil { 94 | return err 95 | } 96 | zerolog.Ctx(ctx).Info().Str("link", link).Msg("Completed build") 97 | 98 | build, err := db.GetBuild(ctx, commit) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | content, err := json.MarshalIndent(&build, "", " ") 104 | if err != nil { 105 | return err 106 | } 107 | 108 | fmt.Printf("%s\n", string(content)) 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /cmd/s3download/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io" 21 | "os" 22 | 23 | "github.com/Netflix/p2plab/downloaders/s3downloader" 24 | "github.com/Netflix/p2plab/pkg/httputil" 25 | "github.com/rs/zerolog" 26 | "github.com/rs/zerolog/log" 27 | ) 28 | 29 | func init() { 30 | // UNIX Time is faster and smaller than most timestamps. If you set 31 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 32 | // time. 33 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 34 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 35 | } 36 | 37 | func main() { 38 | if len(os.Args) != 3 { 39 | fmt.Fprintf(os.Stderr, "s3download: must specify ref and path to save file") 40 | os.Exit(1) 41 | } 42 | 43 | err := run(os.Args[1], os.Args[2]) 44 | if err != nil { 45 | fmt.Fprintf(os.Stderr, "s3download: %s\n", err) 46 | os.Exit(1) 47 | } 48 | } 49 | 50 | func run(ref, filename string) error { 51 | client := httputil.NewHTTPClient() 52 | downloader, err := s3downloader.New(client, s3downloader.S3DownloaderSettings{ 53 | Region: os.Getenv("LABAGENT_DOWNLOADER_S3_REGION"), 54 | }) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger() 60 | ctx := logger.WithContext(context.Background()) 61 | 62 | rc, err := downloader.Download(ctx, ref) 63 | if err != nil { 64 | return err 65 | } 66 | defer rc.Close() 67 | 68 | f, err := os.Create(filename) 69 | if err != nil { 70 | return err 71 | } 72 | defer f.Close() 73 | 74 | n, err := io.Copy(f, rc) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | zerolog.Ctx(ctx).Info().Str("ref", ref).Str("path", filename).Int64("bytes", n).Msg("Completed download") 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /cmd/s3upload/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/Netflix/p2plab/pkg/httputil" 23 | "github.com/Netflix/p2plab/uploaders/s3uploader" 24 | "github.com/rs/zerolog" 25 | "github.com/rs/zerolog/log" 26 | ) 27 | 28 | func init() { 29 | // UNIX Time is faster and smaller than most timestamps. If you set 30 | // zerolog.TimeFieldFormat to an empty string, logs will write with UNIX 31 | // time. 32 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 33 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 34 | } 35 | 36 | func main() { 37 | if len(os.Args) != 2 { 38 | fmt.Fprintf(os.Stderr, "s3upload: must specify path to binary to upload") 39 | os.Exit(1) 40 | } 41 | 42 | err := run(os.Args[1]) 43 | if err != nil { 44 | fmt.Fprintf(os.Stderr, "s3upload: %s\n", err) 45 | os.Exit(1) 46 | } 47 | } 48 | 49 | func run(filename string) error { 50 | client := httputil.NewHTTPClient() 51 | uploader, err := s3uploader.New(client, s3uploader.S3UploaderSettings{ 52 | Bucket: os.Getenv("LABD_UPLOADER_S3_BUCKET"), 53 | Prefix: os.Getenv("LABD_UPLOADER_S3_PREFIX"), 54 | Region: os.Getenv("LABD_UPLOADER_S3_REGION"), 55 | }) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | f, err := os.Open(filename) 61 | if err != nil { 62 | return err 63 | } 64 | defer f.Close() 65 | 66 | logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger() 67 | ctx := logger.WithContext(context.Background()) 68 | ref, err := uploader.Upload(ctx, f) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | zerolog.Ctx(ctx).Info().Str("ref", ref).Str("path", filename).Msg("Completed upload") 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /cue/cue.mod/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build 2 | build: 3 | cue export p2plab_example1.cue p2plab.cue 4 | -------------------------------------------------------------------------------- /cue/cue.mod/module.cue: -------------------------------------------------------------------------------- 1 | module: "p2plab" 2 | -------------------------------------------------------------------------------- /cue/cue.mod/p2plab.cue: -------------------------------------------------------------------------------- 1 | // defines a set of nodes of size 1 or higher 2 | // a "node" is simply an EC2 instance provisioned of the given type 3 | // and there may be more than 1 node in a group, however there must always be 1 4 | Group :: { 5 | // must be greater than or equal to 1 6 | // default value of this field is 1 7 | size: >=1 | *1 8 | instanceType: string 9 | region: string 10 | // labels is an optional field 11 | labels?: [...string] 12 | // although not optional if left unspecified 13 | // then we use the default values of Peer 14 | peer: Peer | *Peer 15 | } 16 | 17 | Peer :: { 18 | gitReference: string | *"HEAD" 19 | transports: [...string] | *["tcp"] 20 | muxers: [...string] | *["mplex"] 21 | securityTransports: [...string] | *["secio"] 22 | routing: string | *"nil" 23 | } 24 | 25 | // a cluster is a collection of 1 or more groups of nodes 26 | // that will be participating in a given benchmark 27 | Cluster :: { 28 | groups: [...Group] 29 | } 30 | 31 | // an object is a particular data format to be used in benchmarking 32 | // typically these are container images 33 | object :: [Name=_]: { 34 | type: string 35 | source: string 36 | } 37 | 38 | Scenario :: { 39 | objects: [...object] 40 | seed: { ... } 41 | // enable any fields for benchmark 42 | benchmark: { ... } 43 | } 44 | 45 | Trial :: { 46 | cluster: Cluster 47 | scenario: Scenario 48 | } 49 | 50 | Experiment :: { 51 | trials: [...Trial] 52 | // trials: [ ...[Cluster,Scenario]] 53 | } -------------------------------------------------------------------------------- /cue/cue.mod/p2plab_example1.cue: -------------------------------------------------------------------------------- 1 | package p2plab 2 | 3 | 4 | items:: object & { 5 | golang: { 6 | type: "oci" 7 | source: "docker.io/library/golang:latest" 8 | } 9 | mysql: { 10 | type: "oci" 11 | source: "docker.io/library/mysql:latest" 12 | } 13 | } 14 | 15 | clust1:: Cluster & { 16 | groups: [ 17 | Group & { 18 | size: 10 19 | instanceType: "t3.micro" 20 | region: "us-west-1" 21 | }, 22 | Group & { 23 | size: 2 24 | instanceType: "t3.medium" 25 | region: "us-east-1" 26 | labels: [ "neighbors" ] 27 | } 28 | ] 29 | } 30 | 31 | scen1:: Scenario & { 32 | objects: [ items ] 33 | seed: { 34 | "neighbors": "golang" 35 | } 36 | benchmark: { 37 | "(not 'neighbors')": "golang" 38 | } 39 | } 40 | 41 | scen2:: Scenario & { 42 | objects: [ items ] 43 | seed: { 44 | "neighbors": "mysql" 45 | } 46 | benchmark: { 47 | "(not 'neighbors')": "mysql" 48 | } 49 | } 50 | 51 | experiment: Experiment & { 52 | trials: [ 53 | Trial & { 54 | cluster: clust1 55 | scenario: scen1 56 | }, 57 | Trial & { 58 | cluster: clust1 59 | scenario: scen2 60 | } 61 | ] 62 | } 63 | -------------------------------------------------------------------------------- /cue/cue.mod/p2plab_example2.cue: -------------------------------------------------------------------------------- 1 | package p2plab 2 | experiment: Experiment & { 3 | trials: [ 4 | Trial & { 5 | cluster: groups: [ 6 | { 7 | size: 1 8 | instanceType: "t3.micro" 9 | region: "us-west-1" 10 | }, 11 | { 12 | size: 2 13 | instanceType: "t3.micro" 14 | region: "us-west-1" 15 | labels: [ "neighbors"] 16 | }, 17 | ] 18 | scenario: { 19 | objects: o 20 | seed: { 21 | "neighbors": "image" 22 | } 23 | benchmark: { 24 | "(not 'neighbors')": "image" 25 | } 26 | } 27 | } for o in objects 28 | ] 29 | } 30 | 31 | objects :: [ 32 | [{ 33 | image: { 34 | type: "oci" 35 | source: "docker.io/library/golang:latest" 36 | } 37 | }], 38 | [{ 39 | image: { 40 | type: "oci" 41 | source: "docker.io/library/mysql:latest" 42 | } 43 | }], 44 | ] 45 | -------------------------------------------------------------------------------- /cue/parser/doc.go: -------------------------------------------------------------------------------- 1 | // Package parser enables parsing of cue sources 2 | package parser 3 | -------------------------------------------------------------------------------- /cue/parser/p2plab_instance.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "cuelang.org/go/cue" 7 | "github.com/Netflix/p2plab/metadata" 8 | ) 9 | 10 | // P2PLabInstance is a wrapper around a cue instance 11 | // which exposes helper functions to reduce lookup verbosity 12 | type P2PLabInstance struct { 13 | *cue.Instance 14 | } 15 | 16 | // ToExperimentDefinition returns the cue source as a metadata.ExperimentDefinition type 17 | func (p *P2PLabInstance) ToExperimentDefinition() (metadata.ExperimentDefinition, error) { 18 | trials, err := p.ToTrialDefinitions() 19 | if err != nil { 20 | return metadata.ExperimentDefinition{}, err 21 | } 22 | return metadata.ExperimentDefinition{ 23 | Trials: trials, 24 | }, nil 25 | } 26 | 27 | // ToTrialDefinitions returns the slice of all trials to be run 28 | func (p *P2PLabInstance) ToTrialDefinitions() ([]metadata.TrialDefinition, error) { 29 | var ( 30 | def = make([]metadata.TrialDefinition, 0) 31 | trials = p.Lookup("experiment").Lookup("trials") 32 | ) 33 | if trials.Err() != nil { 34 | return nil, trials.Err() 35 | } 36 | iter, err := trials.List() 37 | if err != nil { 38 | return nil, err 39 | } 40 | for iter.Next() { 41 | clusterData, err := getJSON( 42 | iter.Value().Lookup("cluster").Lookup("groups"), 43 | ) 44 | if err != nil { 45 | return nil, err 46 | } 47 | objIter, err := iter.Value().Lookup("scenario").Lookup("objects").List() 48 | if err != nil { 49 | return nil, err 50 | } 51 | objects, err := getScenarioObjectDefinition(objIter) 52 | if err != nil { 53 | return nil, err 54 | } 55 | seedData, err := getJSON(iter.Value().Lookup("scenario").Lookup("seed")) 56 | if err != nil { 57 | return nil, err 58 | } 59 | benchData, err := getJSON(iter.Value().Lookup("scenario").Lookup("benchmark")) 60 | if err != nil { 61 | return nil, err 62 | } 63 | trial := metadata.TrialDefinition{ 64 | Scenario: metadata.ScenarioDefinition{ 65 | Objects: objects, 66 | }, 67 | } 68 | if err := json.Unmarshal(clusterData, &trial.Cluster.Groups); err != nil { 69 | return nil, err 70 | } 71 | if err := json.Unmarshal(seedData, &trial.Scenario.Seed); err != nil { 72 | return nil, err 73 | } 74 | if err := json.Unmarshal(benchData, &trial.Scenario.Benchmark); err != nil { 75 | return nil, err 76 | } 77 | def = append(def, trial) 78 | } 79 | return def, nil 80 | } 81 | 82 | // slightly less to type repeatedly 83 | func getJSON(val cue.Value) ([]byte, error) { 84 | return val.MarshalJSON() 85 | } 86 | 87 | func getScenarioObjectDefinition(iter cue.Iterator) (map[string]metadata.ObjectDefinition, error) { 88 | objData, err := getIterData(iter) 89 | if err != nil { 90 | return nil, err 91 | } 92 | var objDef = make(map[string]metadata.ObjectDefinition) 93 | if err := json.Unmarshal(objData, &objDef); err != nil { 94 | return nil, err 95 | } 96 | return objDef, nil 97 | } 98 | 99 | func getIterData(iter cue.Iterator) ([]byte, error) { 100 | var iterData []byte 101 | for iter.Next() { 102 | data, err := getJSON(iter.Value()) 103 | if err != nil { 104 | return nil, err 105 | } 106 | iterData = append(iterData, data...) 107 | } 108 | return iterData, nil 109 | } 110 | -------------------------------------------------------------------------------- /cue/parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "cuelang.org/go/cue" 5 | ) 6 | 7 | var ( 8 | // CueTemplate contains the bare cue source template used to generate 9 | // cue files 10 | CueTemplate = ` 11 | // defines a set of nodes of size 1 or higher 12 | // a "node" is simply an EC2 instance provisioned of the given type 13 | // and there may be more than 1 node in a group, however there must always be 1 14 | Group :: { 15 | // must be greater than or equal to 1 16 | // default value of this field is 1 17 | size: >=1 | *1 18 | instanceType: string 19 | region: string 20 | // labels is an optional field 21 | labels?: [...string] 22 | // although not optional if left unspecified 23 | // then we use the default values of Peer 24 | peer: Peer | *Peer 25 | } 26 | 27 | Peer :: { 28 | gitReference: string | *"HEAD" 29 | transports: [...string] | *["tcp"] 30 | muxers: [...string] | *["mplex"] 31 | securityTransports: [...string] | *["secio"] 32 | routing: string | *"nil" 33 | } 34 | 35 | // a cluster is a collection of 1 or more groups of nodes 36 | // that will be participating in a given benchmark 37 | Cluster :: { 38 | groups: [...Group] 39 | } 40 | 41 | // an object is a particular data format to be used in benchmarking 42 | // typically these are container images 43 | object :: [Name=_]: { 44 | type: string 45 | source: string 46 | } 47 | 48 | Scenario :: { 49 | objects: [...object] 50 | seed: { ... } 51 | // enable any fields for benchmark 52 | benchmark: { ... } 53 | } 54 | 55 | Trial :: { 56 | cluster: Cluster 57 | scenario: Scenario 58 | } 59 | 60 | Experiment :: { 61 | trials: [...Trial] 62 | } 63 | ` 64 | ) 65 | 66 | // Parser bundles the cue runtime with helper functions 67 | // to enable parsing of cue source files 68 | type Parser struct { 69 | entrypoints []string 70 | runtime *cue.Runtime 71 | } 72 | 73 | // NewParser returns a ready to use cue parser 74 | func NewParser(entrypoints []string) *Parser { 75 | p := &Parser{ 76 | // entrypoints are the actual cue source files 77 | // we want to use when building p2plab cue files. 78 | // essentially they are the actual definitions 79 | // used to validate incoming cue source files 80 | entrypoints: entrypoints, 81 | runtime: new(cue.Runtime), 82 | } 83 | return p 84 | } 85 | 86 | // Compile is used to compile the given cue source into our runtime 87 | // it returns a wrapped cue.Instance that provides helper lookup functions 88 | func (p *Parser) Compile(name string, cueSource string) (*P2PLabInstance, error) { 89 | // this is a temporary work around 90 | // until we can properly figure out the cue api 91 | for _, point := range p.entrypoints { 92 | cueSource += point 93 | } 94 | inst, err := p.runtime.Compile(name, cueSource) 95 | if err != nil { 96 | return nil, err 97 | } 98 | return &P2PLabInstance{inst}, nil 99 | } 100 | -------------------------------------------------------------------------------- /cue/parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | ) 7 | 8 | func TestParser(t *testing.T) { 9 | data, err := ioutil.ReadFile("../cue.mod/p2plab.cue") 10 | if err != nil { 11 | t.Fatal(err) 12 | } 13 | parser := NewParser([]string{string(data)}) 14 | var sourceFiles = []string{ 15 | "../cue.mod/p2plab_example1.cue", 16 | "../cue.mod/p2plab_example2.cue", 17 | } 18 | for _, sourceFile := range sourceFiles { 19 | data, err = ioutil.ReadFile(sourceFile) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | pinst, err := parser.Compile("p2plab_example", string(data)) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | _, err = pinst.ToExperimentDefinition() 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /daemon/cancel_closer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package daemon 16 | 17 | type CancelCloser struct { 18 | Cancel func() 19 | } 20 | 21 | func (c *CancelCloser) Close() error { 22 | c.Cancel() 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /daemon/healthcheckrouter/router.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package healthcheckrouter 16 | 17 | import ( 18 | "context" 19 | "net/http" 20 | 21 | "github.com/Netflix/p2plab/daemon" 22 | ) 23 | 24 | type router struct{} 25 | 26 | func New() daemon.Router { 27 | return &router{} 28 | } 29 | 30 | func (s *router) Routes() []daemon.Route { 31 | return []daemon.Route{ 32 | // GET 33 | daemon.NewGetRoute("/healthcheck", s.healthcheck), 34 | } 35 | } 36 | 37 | func (s *router) healthcheck(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { 38 | w.WriteHeader(http.StatusOK) 39 | w.Write([]byte("OK")) 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /daemon/router.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package daemon 16 | 17 | import ( 18 | "context" 19 | "net/http" 20 | ) 21 | 22 | type Router interface { 23 | Routes() []Route 24 | } 25 | 26 | type Route interface { 27 | Method() string 28 | Path() string 29 | Handler() Handler 30 | } 31 | 32 | type Handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error 33 | 34 | type route struct { 35 | method string 36 | path string 37 | handler Handler 38 | } 39 | 40 | func NewRoute(method, path string, handler Handler) Route { 41 | return &route{method, path, handler} 42 | } 43 | 44 | func (r *route) Method() string { 45 | return r.method 46 | } 47 | 48 | func (r *route) Path() string { 49 | return r.path 50 | } 51 | 52 | func (r *route) Handler() Handler { 53 | return r.handler 54 | } 55 | 56 | func NewGetRoute(path string, handler Handler) Route { 57 | return NewRoute("GET", path, handler) 58 | } 59 | 60 | func NewPostRoute(path string, handler Handler) Route { 61 | return NewRoute("POST", path, handler) 62 | } 63 | 64 | func NewPutRoute(path string, handler Handler) Route { 65 | return NewRoute("PUT", path, handler) 66 | } 67 | 68 | func NewDeleteRoute(path string, handler Handler) Route { 69 | return NewRoute("DELETE", path, handler) 70 | } 71 | 72 | func NewOptionsRoute(path string, handler Handler) Route { 73 | return NewRoute("OPTIONS", path, handler) 74 | } 75 | 76 | func NewHeadRoute(path string, handler Handler) Route { 77 | return NewRoute("HEAD", path, handler) 78 | } 79 | -------------------------------------------------------------------------------- /dag/walker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package dag 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/pkg/traceutil" 21 | cid "github.com/ipfs/go-cid" 22 | ipld "github.com/ipfs/go-ipld-format" 23 | opentracing "github.com/opentracing/opentracing-go" 24 | "golang.org/x/sync/errgroup" 25 | ) 26 | 27 | func Walk(ctx context.Context, c cid.Cid, ng ipld.NodeGetter) error { 28 | span, ctx := traceutil.StartSpanFromContext(ctx, "dag.Walk") 29 | defer span.Finish() 30 | span.SetTag("cid", c.String()) 31 | 32 | nd, err := ng.Get(ctx, c) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | return walk(ctx, nd, ng) 38 | } 39 | 40 | func walk(ctx context.Context, nd ipld.Node, ng ipld.NodeGetter) error { 41 | var cids []cid.Cid 42 | for _, link := range nd.Links() { 43 | cids = append(cids, link.Cid) 44 | } 45 | 46 | if len(cids) > 0 { 47 | var span opentracing.Span 48 | span, ctx = traceutil.StartSpanFromContext(ctx, "dag.walk") 49 | defer span.Finish() 50 | span.SetTag("cid", nd.Cid().String()) 51 | } 52 | 53 | eg, gctx := errgroup.WithContext(ctx) 54 | 55 | ndChan := ng.GetMany(ctx, cids) 56 | for ndOpt := range ndChan { 57 | if ndOpt.Err != nil { 58 | return ndOpt.Err 59 | } 60 | 61 | nd := ndOpt.Node 62 | eg.Go(func() error { 63 | return walk(gctx, nd, ng) 64 | }) 65 | } 66 | 67 | err := eg.Wait() 68 | if err != nil { 69 | return err 70 | } 71 | 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /distribution.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | "io" 20 | 21 | "github.com/Netflix/p2plab/metadata" 22 | ) 23 | 24 | // Builder compiles the peer to hotswap the underlying implementation on a live 25 | // cluster. 26 | type Builder interface { 27 | // Init initializes the builder. 28 | Init(ctx context.Context) error 29 | 30 | // Resolve resolves a git-ref to a commit it can uniquely build. 31 | Resolve(ctx context.Context, ref string) (commit string, err error) 32 | 33 | // Build compiles the peer at the given commit. 34 | Build(ctx context.Context, commit string) (link string, err error) 35 | } 36 | 37 | // BuildAPI defines the API for build operations. 38 | type BuildAPI interface { 39 | // Get returns a build. 40 | Get(ctx context.Context, id string) (Build, error) 41 | 42 | // List returns available builds. 43 | List(ctx context.Context) ([]Build, error) 44 | 45 | // Upload uploads a binary for a build. 46 | Upload(ctx context.Context, r io.Reader) (Build, error) 47 | } 48 | 49 | // Build is an compiled peer ready to be deployed. 50 | type Build interface { 51 | // ID returns a uniquely identifiable string. 52 | ID() string 53 | 54 | Metadata() metadata.Build 55 | 56 | // Open creates a reader for the build's binary. 57 | Open(ctx context.Context) (io.ReadCloser, error) 58 | } 59 | 60 | // Uploader uploads artifacts to an external distribution mechanism. 61 | type Uploader interface { 62 | // Upload uploads artifacts to a registry. 63 | Upload(ctx context.Context, r io.Reader) (link string, err error) 64 | 65 | Close() error 66 | } 67 | 68 | // Download downlaods artifacts from an external distribution mechanism. 69 | type Downloader interface { 70 | // Download downloads an artifact with an abstract link. 71 | Download(ctx context.Context, link string) (io.ReadCloser, error) 72 | } 73 | -------------------------------------------------------------------------------- /downloaders/downloaders.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloaders 16 | 17 | import ( 18 | "sync" 19 | 20 | "github.com/Netflix/p2plab" 21 | "github.com/Netflix/p2plab/downloaders/filedownloader" 22 | "github.com/Netflix/p2plab/downloaders/httpdownloader" 23 | "github.com/Netflix/p2plab/downloaders/s3downloader" 24 | "github.com/Netflix/p2plab/pkg/httputil" 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | type DownloaderSettings struct { 29 | Client *httputil.Client 30 | 31 | S3 s3downloader.S3DownloaderSettings 32 | } 33 | 34 | type Downloaders struct { 35 | root string 36 | settings DownloaderSettings 37 | mu sync.Mutex 38 | fs map[string]p2plab.Downloader 39 | } 40 | 41 | func New(root string, settings DownloaderSettings) *Downloaders { 42 | return &Downloaders{ 43 | root: root, 44 | settings: settings, 45 | fs: make(map[string]p2plab.Downloader), 46 | } 47 | } 48 | 49 | func (f *Downloaders) Get(downloaderType string) (p2plab.Downloader, error) { 50 | f.mu.Lock() 51 | defer f.mu.Unlock() 52 | 53 | downloader, ok := f.fs[downloaderType] 54 | if !ok { 55 | var err error 56 | downloader, err = f.newDownloader(downloaderType) 57 | if err != nil { 58 | return nil, err 59 | } 60 | f.fs[downloaderType] = downloader 61 | } 62 | return downloader, nil 63 | } 64 | 65 | func (f *Downloaders) newDownloader(downloaderType string) (p2plab.Downloader, error) { 66 | // root := filepath.Join(f.root, downloaderType) 67 | switch downloaderType { 68 | case "file": 69 | return filedownloader.New(), nil 70 | case "http", "https": 71 | return httpdownloader.New(f.settings.Client), nil 72 | case "s3": 73 | return s3downloader.New(f.settings.Client.HTTPClient, f.settings.S3) 74 | default: 75 | return nil, errors.Errorf("unrecognized downloader type: %q", downloaderType) 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /downloaders/filedownloader/downloader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filedownloader 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "net/url" 21 | "os" 22 | "path/filepath" 23 | 24 | "github.com/Netflix/p2plab" 25 | "github.com/Netflix/p2plab/errdefs" 26 | "github.com/pkg/errors" 27 | ) 28 | 29 | type downloader struct { 30 | } 31 | 32 | func New() p2plab.Downloader { 33 | return &downloader{} 34 | } 35 | 36 | func (f *downloader) Download(ctx context.Context, link string) (io.ReadCloser, error) { 37 | u, err := url.Parse(link) 38 | if err != nil { 39 | return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "invalid url %q", link) 40 | } 41 | 42 | downloadPath := filepath.Join(u.Host, u.Path) 43 | return os.Open(downloadPath) 44 | } 45 | -------------------------------------------------------------------------------- /downloaders/httpdownloader/downloader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package httpdownloader 16 | 17 | import ( 18 | "context" 19 | "io" 20 | 21 | "github.com/Netflix/p2plab" 22 | "github.com/Netflix/p2plab/pkg/httputil" 23 | ) 24 | 25 | type downloader struct { 26 | client *httputil.Client 27 | } 28 | 29 | func New(client *httputil.Client) p2plab.Downloader { 30 | return &downloader{client} 31 | } 32 | 33 | func (f *downloader) Download(ctx context.Context, link string) (io.ReadCloser, error) { 34 | req := f.client.NewRequest("GET", link) 35 | resp, err := req.Send(ctx) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | return resp.Body, nil 41 | } 42 | -------------------------------------------------------------------------------- /downloaders/s3downloader/downloader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package s3downloader 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "io" 21 | "io/ioutil" 22 | "net/http" 23 | "net/url" 24 | "time" 25 | 26 | "github.com/Netflix/p2plab" 27 | "github.com/Netflix/p2plab/errdefs" 28 | "github.com/Netflix/p2plab/pkg/logutil" 29 | "github.com/aws/aws-sdk-go-v2/aws" 30 | "github.com/aws/aws-sdk-go-v2/aws/external" 31 | "github.com/aws/aws-sdk-go-v2/service/s3" 32 | "github.com/aws/aws-sdk-go-v2/service/s3/s3manager" 33 | "github.com/pkg/errors" 34 | "github.com/rs/zerolog" 35 | ) 36 | 37 | type S3DownloaderSettings struct { 38 | Region string 39 | } 40 | 41 | type downloader struct { 42 | downloadManager *s3manager.Downloader 43 | } 44 | 45 | func New(client *http.Client, settings S3DownloaderSettings) (p2plab.Downloader, error) { 46 | cfg, err := external.LoadDefaultAWSConfig() 47 | if err != nil { 48 | return nil, errors.Wrap(err, "failed to load aws config") 49 | } 50 | cfg.Region = settings.Region 51 | cfg.HTTPClient = client 52 | 53 | return &downloader{ 54 | downloadManager: s3manager.NewDownloader(cfg), 55 | }, nil 56 | } 57 | 58 | func (f *downloader) Download(ctx context.Context, link string) (io.ReadCloser, error) { 59 | u, err := url.Parse(link) 60 | if err != nil { 61 | return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "invalid url %q", link) 62 | } 63 | bucket := u.Host 64 | 65 | if len(u.Path) == 0 { 66 | return nil, errors.Wrap(errdefs.ErrInvalidArgument, "zero length s3 key") 67 | } 68 | key := u.Path[1:] 69 | 70 | logger := zerolog.Ctx(ctx).With().Str("bucket", bucket).Str("key", key).Logger() 71 | ectx, cancel := context.WithCancel(logger.WithContext(ctx)) 72 | defer cancel() 73 | 74 | go logutil.Elapsed(ectx, 20*time.Second, "Downloading S3 object") 75 | 76 | buf := aws.NewWriteAtBuffer([]byte{}) 77 | n, err := f.downloadManager.DownloadWithContext(ctx, buf, &s3.GetObjectInput{ 78 | Bucket: aws.String(bucket), 79 | Key: aws.String(key), 80 | }) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | zerolog.Ctx(ctx).Debug().Int64("bytes", n).Msg("Downloaded object from S3") 86 | return ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil 87 | } 88 | -------------------------------------------------------------------------------- /errdefs/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errdefs 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/pkg/errors" 21 | ) 22 | 23 | var ( 24 | // ErrAlreadyExists is returned when a resource already exists. 25 | ErrAlreadyExists = errors.New("already exists") 26 | 27 | // ErrNotFound is returned when a resource is not found. 28 | ErrNotFound = errors.New("not found") 29 | 30 | // ErrInvalidArgument is returned when a invalid argument was given. 31 | ErrInvalidArgument = errors.New("invalid argument") 32 | 33 | ErrUnavailable = errors.New("unavailable") 34 | ) 35 | 36 | func IsAlreadyExists(err error) bool { 37 | return errors.Cause(err) == ErrAlreadyExists 38 | } 39 | 40 | func IsNotFound(err error) bool { 41 | return errors.Cause(err) == ErrNotFound 42 | } 43 | 44 | func IsInvalidArgument(err error) bool { 45 | return errors.Cause(err) == ErrInvalidArgument 46 | } 47 | 48 | func IsUnavailable(err error) bool { 49 | return errors.Cause(err) == ErrUnavailable 50 | } 51 | 52 | func IsCancelled(err error) bool { 53 | return errors.Cause(err) == context.Canceled 54 | } 55 | -------------------------------------------------------------------------------- /examples/cluster/multiple-regions.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": [ 3 | { 4 | "size": 3, 5 | "instanceType": "t2.micro", 6 | "region": "us-west-2" 7 | }, 8 | { 9 | "size": 3, 10 | "instanceType": "t2.micro", 11 | "region": "us-east-1" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /examples/cluster/quic.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": [ 3 | { 4 | "size": 1, 5 | "instanceType": "t2.micro", 6 | "region": "us-west-2", 7 | "peer": { 8 | "gitReference": "HEAD", 9 | "transports": ["quic"], 10 | "muxers": ["mplex"], 11 | "securityOptions": [], 12 | "routing": "nil" 13 | } 14 | }, 15 | { 16 | "size": 2, 17 | "instanceType": "t2.micro", 18 | "region": "us-west-2", 19 | "labels": ["neighbors"], 20 | "peer": { 21 | "gitReference": "HEAD", 22 | "transports": ["quic"], 23 | "muxers": ["mplex"], 24 | "securityOptions": [], 25 | "routing": "nil" 26 | } 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /examples/cluster/same-region.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": [ 3 | { 4 | "size": 1, 5 | "instanceType": "t2.micro", 6 | "region": "us-west-2" 7 | }, 8 | { 9 | "size": 2, 10 | "instanceType": "t2.micro", 11 | "region": "us-west-2", 12 | "labels": ["neighbors"] 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /examples/peer/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "gitReference": "HEAD", 3 | "transports": ["tcp"], 4 | "muxers": ["mplex"], 5 | "securityTransports": ["secio"], 6 | "routing": "nil" 7 | } 8 | -------------------------------------------------------------------------------- /examples/scenario/cross-region.json: -------------------------------------------------------------------------------- 1 | { 2 | "objects": { 3 | "golang": { 4 | "type": "oci", 5 | "source": "docker.io/library/golang:latest" 6 | } 7 | }, 8 | "seed": { 9 | "us-east-1": "golang" 10 | }, 11 | "benchmark": { 12 | "us-west-2": "golang" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/scenario/image-deduplication.json: -------------------------------------------------------------------------------- 1 | { 2 | "objects": { 3 | "ubuntu-v1": { 4 | "type": "oci", 5 | "source": "docker.io/library/ubuntu:xenial-20190610" 6 | }, 7 | "ubuntu-v2": { 8 | "type": "oci", 9 | "source": "docker.io/library/ubuntu:xenial-20190720" 10 | } 11 | }, 12 | "seed": { 13 | "neighbors": "ubuntu-v2", 14 | "(not 'neighbors')": "ubuntu-v1" 15 | }, 16 | "benchmark": { 17 | "(not 'neighbors')": "ubuntu-v2" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /examples/scenario/neighbors.json: -------------------------------------------------------------------------------- 1 | { 2 | "objects": { 3 | "golang": { 4 | "type": "oci", 5 | "source": "docker.io/library/golang:latest" 6 | } 7 | }, 8 | "seed": { 9 | "neighbors": "golang" 10 | }, 11 | "benchmark": { 12 | "(not 'neighbors')": "golang" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /experiment.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | // ExperimentAPI is an unimplemented layer to run experiments, a collection 24 | // of benchmarks while varying some aspect. 25 | type ExperimentAPI interface { 26 | Create(ctx context.Context, name string, edef metadata.ExperimentDefinition) (id string, err error) 27 | 28 | Get(ctx context.Context, id string) (Experiment, error) 29 | 30 | Label(ctx context.Context, ids, adds, removes []string) ([]Experiment, error) 31 | 32 | List(ctx context.Context, opts ...ListOption) ([]Experiment, error) 33 | 34 | Remove(ctx context.Context, ids ...string) error 35 | } 36 | 37 | type Experiment interface { 38 | Labeled 39 | 40 | Metadata() metadata.Experiment 41 | } 42 | -------------------------------------------------------------------------------- /experiments/definition.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package experiments 16 | 17 | import ( 18 | "io/ioutil" 19 | 20 | "github.com/Netflix/p2plab/cue/parser" 21 | "github.com/Netflix/p2plab/metadata" 22 | ) 23 | 24 | // Parse reads the cue source file at filename and converts it to a 25 | // metadata.ExperimentDefinition type 26 | func Parse(filename string) (metadata.ExperimentDefinition, error) { 27 | content, err := ioutil.ReadFile(filename) 28 | if err != nil { 29 | return metadata.ExperimentDefinition{}, err 30 | } 31 | psr := parser.NewParser([]string{parser.CueTemplate}) 32 | inst, err := psr.Compile( 33 | "p2plab_experiment", 34 | string(content), 35 | ) 36 | if err != nil { 37 | return metadata.ExperimentDefinition{}, err 38 | } 39 | return inst.ToExperimentDefinition() 40 | } 41 | -------------------------------------------------------------------------------- /experiments/experiments_test.go: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/Netflix/p2plab/metadata" 10 | "github.com/google/uuid" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestExperimentDefinition(t *testing.T) { 15 | ctx, cancel := context.WithCancel(context.Background()) 16 | defer cancel() 17 | db, cleanup := newTestDB(t, "exptestdir") 18 | defer func() { 19 | require.NoError(t, cleanup()) 20 | }() 21 | var ids []string 22 | t.Run("Experiment Creation And Retrieval", func(t *testing.T) { 23 | sourceFiles := []string{ 24 | "../cue/cue.mod/p2plab_example1.cue", 25 | "../cue/cue.mod/p2plab_example2.cue", 26 | } 27 | for _, sourceFile := range sourceFiles { 28 | name := strings.Split(sourceFile, "/") 29 | exp1 := newTestExperiment(t, sourceFile, name[len(name)-1]) 30 | ids = append(ids, exp1.ID) 31 | exp2, err := db.CreateExperiment(ctx, exp1) 32 | require.NoError(t, err) 33 | require.Equal(t, exp1.ID, exp2.ID) 34 | require.Equal(t, exp1.Status, exp2.Status) 35 | require.Equal(t, exp1.Definition, exp2.Definition) 36 | exp3, err := db.GetExperiment(ctx, exp1.ID) 37 | require.NoError(t, err) 38 | require.Equal(t, exp1.ID, exp3.ID) 39 | require.Equal(t, exp1.Status, exp3.Status) 40 | require.Equal(t, exp1.Definition, exp3.Definition) 41 | } 42 | }) 43 | t.Run("List Experiments", func(t *testing.T) { 44 | experiments, err := db.ListExperiments(ctx) 45 | require.NoError(t, err) 46 | for _, experiment := range experiments { 47 | if experiment.ID != ids[0] && experiment.ID != ids[1] { 48 | t.Error("bad experiment id found") 49 | } 50 | } 51 | }) 52 | t.Run("Update Experiments", func(t *testing.T) { 53 | for _, id := range ids { 54 | exp, err := db.GetExperiment(ctx, id) 55 | require.NoError(t, err) 56 | prevUpdateAt := exp.UpdatedAt 57 | exp.Labels = append(exp.Labels, "test label") 58 | exp, err = db.UpdateExperiment(ctx, exp) 59 | require.NoError(t, err) 60 | require.True(t, exp.UpdatedAt.After(prevUpdateAt)) 61 | } 62 | }) 63 | t.Run("Label Experiments", func(t *testing.T) { 64 | exps, err := db.LabelExperiments( 65 | ctx, 66 | ids, 67 | []string{"should be present"}, 68 | []string{"test label"}, 69 | ) 70 | require.NoError(t, err) 71 | for _, exp := range exps { 72 | require.Len(t, exp.Labels, 1) 73 | require.Equal(t, exp.Labels[0], "should be present") 74 | } 75 | }) 76 | t.Run("Delete Experiment", func(t *testing.T) { 77 | require.NoError(t, db.DeleteExperiment(ctx, ids[0])) 78 | _, err := db.GetExperiment(ctx, ids[0]) 79 | require.Error(t, err) 80 | _, err = db.GetExperiment(ctx, ids[1]) 81 | require.NoError(t, err) 82 | }) 83 | } 84 | 85 | func newTestExperiment(t *testing.T, sourceFile, name string) metadata.Experiment { 86 | edef, err := Parse(sourceFile) 87 | require.NoError(t, err) 88 | return metadata.Experiment{ 89 | ID: uuid.New().String(), 90 | Status: metadata.ExperimentRunning, 91 | Definition: edef, 92 | } 93 | } 94 | 95 | func newTestDB(t *testing.T, path string) (metadata.DB, func() error) { 96 | db, err := metadata.NewDB(path) 97 | require.NoError(t, err) 98 | cleanup := func() error { 99 | if err := db.Close(); err != nil { 100 | return err 101 | } 102 | return os.RemoveAll(path) 103 | } 104 | return db, cleanup 105 | } 106 | -------------------------------------------------------------------------------- /experiments/report.go: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/Netflix/p2plab/metadata" 9 | ) 10 | 11 | // ReportToCSV takes a slice of metadata reports and converts it to csv 12 | func ReportToCSV(reports []metadata.Report, output io.Writer) error { 13 | w := csv.NewWriter(output) 14 | columns := [][]string{ 15 | { 16 | "report_number", 17 | "total_time", 18 | // TODO: when testing against a large number of nodes, csv programs report errors about too much data in a single cell 19 | // TODO: this means we need to find a better way of representing the nodes involved in a single trial 20 | // "nodes", 21 | "bitswap_blocks_received", 22 | "bitswap_data_received", 23 | "bitswap_blocks_sent", 24 | "bitswap_data_sent", 25 | "bitswap_dupe_blocks_received", 26 | "bitswap_dupe_data_received", 27 | "bitswap_messages_received", 28 | "bandwidth_total_in", 29 | "bandwidth_total_out", 30 | "bandwidth_rate_in", 31 | "bandwidth_rate_out", 32 | }, 33 | } 34 | for i, report := range reports { 35 | /* see todo at top of function 36 | var nodes string 37 | for node := range report.Nodes { 38 | nodes += fmt.Sprintf("%s%s ", nodes, node) 39 | } 40 | */ 41 | columns = append(columns, []string{ 42 | fmt.Sprint(i), 43 | report.Summary.TotalTime.String(), 44 | // see todo at top of function 45 | // nodes, 46 | fmt.Sprint(report.Aggregates.Totals.Bitswap.BlocksReceived), 47 | fmt.Sprint(report.Aggregates.Totals.Bitswap.DataReceived), 48 | fmt.Sprint(report.Aggregates.Totals.Bitswap.BlocksSent), 49 | fmt.Sprint(report.Aggregates.Totals.Bitswap.DataSent), 50 | fmt.Sprint(report.Aggregates.Totals.Bitswap.DupBlksReceived), 51 | fmt.Sprint(report.Aggregates.Totals.Bitswap.DupDataReceived), 52 | fmt.Sprint(report.Aggregates.Totals.Bitswap.MessagesReceived), 53 | fmt.Sprint(report.Aggregates.Totals.Bandwidth.Totals.TotalIn), 54 | fmt.Sprint(report.Aggregates.Totals.Bandwidth.Totals.TotalOut), 55 | fmt.Sprint(report.Aggregates.Totals.Bandwidth.Totals.RateIn), 56 | fmt.Sprint(report.Aggregates.Totals.Bandwidth.Totals.RateOut), 57 | }) 58 | } 59 | w.WriteAll(columns) 60 | w.Flush() 61 | return w.Error() 62 | } 63 | -------------------------------------------------------------------------------- /experiments/report_test.go: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | metrics "github.com/libp2p/go-libp2p-core/metrics" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/Netflix/p2plab/metadata" 12 | ) 13 | 14 | func TestReportToCSV(t *testing.T) { 15 | testFile := "report.csv" 16 | t.Cleanup(func() { 17 | require.NoError(t, os.Remove(testFile)) 18 | }) 19 | reports := []metadata.Report{ 20 | { 21 | Summary: metadata.ReportSummary{ 22 | TotalTime: time.Hour, 23 | }, 24 | Aggregates: metadata.ReportAggregates{ 25 | Totals: metadata.ReportNode{ 26 | Bitswap: metadata.ReportBitswap{ 27 | BlocksReceived: 1, 28 | DataReceived: 2, 29 | BlocksSent: 3, 30 | DataSent: 4, 31 | DupBlksReceived: 5, 32 | DupDataReceived: 6, 33 | MessagesReceived: 7, 34 | }, 35 | Bandwidth: metadata.ReportBandwidth{ 36 | Totals: metrics.Stats{}, 37 | }, 38 | }, 39 | }, 40 | Nodes: map[string]metadata.ReportNode{ 41 | "node1": {}, 42 | "node2": {}, 43 | "node3": {}, 44 | }, 45 | }, 46 | } 47 | fh, err := os.Create(testFile) 48 | require.NoError(t, err) 49 | defer fh.Close() 50 | require.NoError(t, ReportToCSV(reports, fh)) 51 | } 52 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Netflix/p2plab 2 | 3 | go 1.14 4 | 5 | require ( 6 | cuelang.org/go v0.1.2 7 | github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect 8 | github.com/Microsoft/go-winio v0.4.13-0.20190408173621-84b4ab48a507 // indirect 9 | github.com/Microsoft/hcsshim v0.8.6 // indirect 10 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 11 | github.com/aws/aws-sdk-go-v2 v0.11.0 12 | github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect 13 | github.com/containerd/containerd v1.3.0 14 | github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect 15 | github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible // indirect 16 | github.com/dustin/go-humanize v1.0.0 17 | github.com/fsnotify/fsnotify v1.4.9 // indirect 18 | github.com/gobwas/glob v0.2.3 19 | github.com/google/uuid v1.1.1 20 | github.com/gorilla/mux v1.7.3 21 | github.com/hako/durafmt v0.0.0-20190612201238-650ed9f29a84 22 | github.com/hashicorp/go-cleanhttp v0.5.0 23 | github.com/hashicorp/go-retryablehttp v0.5.4 24 | github.com/ipfs/go-bitswap v0.2.5 25 | github.com/ipfs/go-blockservice v0.1.2 26 | github.com/ipfs/go-cid v0.0.5 27 | github.com/ipfs/go-datastore v0.4.4 28 | github.com/ipfs/go-ds-badger v0.2.1 29 | github.com/ipfs/go-ipfs-blockstore v0.1.4 30 | github.com/ipfs/go-ipfs-chunker v0.0.4 31 | github.com/ipfs/go-ipfs-files v0.0.7 32 | github.com/ipfs/go-ipfs-provider v0.4.1 33 | github.com/ipfs/go-ipfs-routing v0.1.0 34 | github.com/ipfs/go-ipfs-util v0.0.1 35 | github.com/ipfs/go-ipld-cbor v0.0.4 36 | github.com/ipfs/go-ipld-format v0.0.2 37 | github.com/ipfs/go-merkledag v0.3.1 38 | github.com/ipfs/go-unixfs v0.2.4 39 | github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect 40 | github.com/libp2p/go-libp2p v0.6.1 41 | github.com/libp2p/go-libp2p-core v0.5.0 42 | github.com/libp2p/go-libp2p-kad-dht v0.5.2 43 | github.com/libp2p/go-libp2p-mplex v0.2.2 44 | github.com/libp2p/go-libp2p-protocol v0.1.0 45 | github.com/libp2p/go-libp2p-quic-transport v0.3.1 46 | github.com/libp2p/go-libp2p-secio v0.2.1 47 | github.com/libp2p/go-libp2p-swarm v0.2.2 48 | github.com/libp2p/go-libp2p-tls v0.1.3 49 | github.com/libp2p/go-libp2p-yamux v0.2.5 50 | github.com/libp2p/go-maddr-filter v0.0.5 51 | github.com/libp2p/go-tcp-transport v0.1.1 52 | github.com/libp2p/go-ws-transport v0.2.0 53 | github.com/mattn/go-runewidth v0.0.8 // indirect 54 | github.com/multiformats/go-multiaddr v0.2.1 55 | github.com/multiformats/go-multihash v0.0.13 56 | github.com/olekukonko/tablewriter v0.0.2-0.20190618033246-cc27d85e17ce 57 | github.com/opencontainers/go-digest v1.0.0-rc1 58 | github.com/opencontainers/image-spec v1.0.1 59 | github.com/opencontainers/runc v1.0.0-rc8 // indirect 60 | github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 61 | github.com/opentracing/opentracing-go v1.1.0 62 | github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 63 | github.com/pkg/errors v0.9.1 64 | github.com/rs/xid v1.2.1 65 | github.com/rs/zerolog v1.14.4-0.20190719171043-b806a5ecbe53 66 | github.com/sirupsen/logrus v1.4.2 // indirect 67 | github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect 68 | github.com/stretchr/testify v1.5.1 69 | github.com/uber/jaeger-client-go v2.22.1+incompatible 70 | github.com/uber/jaeger-lib v2.0.0+incompatible // indirect 71 | github.com/urfave/cli v1.20.0 72 | go.etcd.io/bbolt v1.3.3 73 | go.uber.org/multierr v1.4.0 // indirect 74 | golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 // indirect 75 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect 76 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect 77 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e 78 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect 79 | gopkg.in/yaml.v2 v2.2.5 // indirect 80 | gotest.tools v2.2.0+incompatible // indirect 81 | ) 82 | -------------------------------------------------------------------------------- /labagent/agentapi/agentapi.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package agentapi 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "encoding/json" 21 | "fmt" 22 | "time" 23 | 24 | "github.com/Netflix/p2plab" 25 | "github.com/Netflix/p2plab/metadata" 26 | "github.com/Netflix/p2plab/pkg/httputil" 27 | "github.com/Netflix/p2plab/pkg/logutil" 28 | "github.com/rs/zerolog" 29 | ) 30 | 31 | type api struct { 32 | addr string 33 | client *httputil.Client 34 | } 35 | 36 | func New(client *httputil.Client, addr string) p2plab.AgentAPI { 37 | return &api{ 38 | addr: addr, 39 | client: client, 40 | } 41 | } 42 | 43 | func (a *api) url(endpoint string, v ...interface{}) string { 44 | return fmt.Sprintf("%s%s", a.addr, fmt.Sprintf(endpoint, v...)) 45 | } 46 | 47 | func (a *api) Healthcheck(ctx context.Context) bool { 48 | req := a.client.NewRequest("GET", a.url("/healthcheck"), 49 | httputil.WithRetryWaitMax(5*time.Minute), 50 | httputil.WithRetryMax(10), 51 | ) 52 | resp, err := req.Send(ctx) 53 | if err != nil { 54 | zerolog.Ctx(ctx).Debug().Str("err", err.Error()).Str("addr", a.addr).Msg("unhealthy") 55 | return false 56 | } 57 | defer resp.Body.Close() 58 | 59 | return true 60 | } 61 | 62 | func (a *api) Update(ctx context.Context, id, link string, pdef metadata.PeerDefinition) error { 63 | content, err := json.MarshalIndent(&pdef, "", " ") 64 | if err != nil { 65 | return err 66 | } 67 | 68 | req := a.client.NewRequest("PUT", a.url("/update")). 69 | Body(bytes.NewReader(content)). 70 | Option("id", id). 71 | Option("link", link) 72 | 73 | resp, err := req.Send(ctx) 74 | if err != nil { 75 | return err 76 | } 77 | defer resp.Body.Close() 78 | 79 | logWriter := logutil.LogWriter(ctx) 80 | if logWriter != nil { 81 | err = logutil.WriteRemoteLogs(ctx, resp.Body, logWriter) 82 | if err != nil { 83 | return err 84 | } 85 | } 86 | 87 | return nil 88 | } 89 | 90 | func (a *api) SSH(ctx context.Context, opts ...p2plab.SSHOption) error { 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /labagent/agentrouter/router.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package agentrouter 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "net/http" 21 | "time" 22 | 23 | "github.com/Netflix/p2plab/daemon" 24 | "github.com/Netflix/p2plab/labagent/supervisor" 25 | "github.com/Netflix/p2plab/metadata" 26 | "github.com/Netflix/p2plab/pkg/logutil" 27 | "github.com/rs/zerolog" 28 | ) 29 | 30 | type router struct { 31 | addr string 32 | supervisor supervisor.Supervisor 33 | } 34 | 35 | func New(addr string, s supervisor.Supervisor) daemon.Router { 36 | return &router{addr, s} 37 | } 38 | 39 | func (s *router) Routes() []daemon.Route { 40 | return []daemon.Route{ 41 | // PUT 42 | daemon.NewPutRoute("/update", s.putUpdate), 43 | } 44 | } 45 | 46 | func (s *router) putUpdate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { 47 | id := r.FormValue("id") 48 | link := r.FormValue("link") 49 | ctx, logger := logutil.WithResponseLogger(ctx, w) 50 | logger.UpdateContext(func(c zerolog.Context) zerolog.Context { 51 | return c.Str("id", id).Str("link", link) 52 | }) 53 | 54 | var pdef metadata.PeerDefinition 55 | err := json.NewDecoder(r.Body).Decode(&pdef) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | err = s.supervisor.Supervise(ctx, id, link, pdef) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | // Give supervised process time to accept network connections. 66 | time.Sleep(time.Second) 67 | 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /labagent/labagent.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package labagent 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "path/filepath" 21 | 22 | "github.com/Netflix/p2plab/daemon" 23 | "github.com/Netflix/p2plab/daemon/healthcheckrouter" 24 | "github.com/Netflix/p2plab/downloaders" 25 | "github.com/Netflix/p2plab/labagent/agentrouter" 26 | "github.com/Netflix/p2plab/labagent/supervisor" 27 | "github.com/Netflix/p2plab/pkg/httputil" 28 | "github.com/rs/zerolog" 29 | ) 30 | 31 | type LabAgent struct { 32 | daemon *daemon.Daemon 33 | closers []io.Closer 34 | } 35 | 36 | func New(root, addr, appRoot, appAddr string, logger *zerolog.Logger, opts ...LabagentOption) (*LabAgent, error) { 37 | var settings LabagentSettings 38 | for _, opt := range opts { 39 | err := opt(&settings) 40 | if err != nil { 41 | return nil, err 42 | } 43 | } 44 | 45 | client, err := httputil.NewClient(httputil.NewHTTPClient(), httputil.WithLogger(logger)) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | settings.DownloaderSettings.Client = client 51 | fs := downloaders.New(filepath.Join(root, "downloaders"), settings.DownloaderSettings) 52 | 53 | s, err := supervisor.New(filepath.Join(root, "supervisor"), appRoot, appAddr, client, fs) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | var closers []io.Closer 59 | daemon, err := daemon.New("labagent", addr, logger, 60 | healthcheckrouter.New(), 61 | agentrouter.New(appAddr, s), 62 | ) 63 | if err != nil { 64 | return nil, err 65 | } 66 | closers = append(closers, daemon) 67 | 68 | return &LabAgent{ 69 | daemon: daemon, 70 | closers: closers, 71 | }, nil 72 | } 73 | 74 | func (a *LabAgent) Close() error { 75 | for _, closer := range a.closers { 76 | err := closer.Close() 77 | if err != nil { 78 | return err 79 | } 80 | } 81 | return nil 82 | } 83 | 84 | func (a *LabAgent) Serve(ctx context.Context) error { 85 | return a.daemon.Serve(ctx) 86 | } 87 | -------------------------------------------------------------------------------- /labagent/settings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package labagent 16 | 17 | import "github.com/Netflix/p2plab/downloaders" 18 | 19 | type LabagentOption func(*LabagentSettings) error 20 | 21 | type LabagentSettings struct { 22 | DownloaderSettings downloaders.DownloaderSettings 23 | } 24 | 25 | func WithDownloaderSettings(settings downloaders.DownloaderSettings) LabagentOption { 26 | return func(s *LabagentSettings) error { 27 | s.DownloaderSettings = settings 28 | return nil 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /labapp/appapi/appapi.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by apilicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package appapi 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "encoding/json" 21 | "fmt" 22 | 23 | "github.com/Netflix/p2plab" 24 | "github.com/Netflix/p2plab/metadata" 25 | "github.com/Netflix/p2plab/pkg/httputil" 26 | "github.com/Netflix/p2plab/pkg/logutil" 27 | "github.com/libp2p/go-libp2p-core/peer" 28 | ) 29 | 30 | type api struct { 31 | addr string 32 | client *httputil.Client 33 | } 34 | 35 | func New(client *httputil.Client, addr string) p2plab.AppAPI { 36 | return &api{ 37 | addr: addr, 38 | client: client, 39 | } 40 | } 41 | 42 | func (a *api) url(endpoint string, v ...interface{}) string { 43 | return fmt.Sprintf("%s%s", a.addr, fmt.Sprintf(endpoint, v...)) 44 | } 45 | 46 | func (a *api) PeerInfo(ctx context.Context) (peer.AddrInfo, error) { 47 | var peerInfo peer.AddrInfo 48 | 49 | req := a.client.NewRequest("GET", a.url("/peerInfo")) 50 | resp, err := req.Send(ctx) 51 | if err != nil { 52 | return peerInfo, err 53 | } 54 | defer resp.Body.Close() 55 | 56 | err = json.NewDecoder(resp.Body).Decode(&peerInfo) 57 | if err != nil { 58 | return peerInfo, err 59 | } 60 | 61 | return peerInfo, nil 62 | } 63 | 64 | func (a *api) Report(ctx context.Context) (metadata.ReportNode, error) { 65 | var report metadata.ReportNode 66 | 67 | req := a.client.NewRequest("GET", a.url("/report")) 68 | resp, err := req.Send(ctx) 69 | if err != nil { 70 | return report, err 71 | } 72 | defer resp.Body.Close() 73 | 74 | err = json.NewDecoder(resp.Body).Decode(&report) 75 | if err != nil { 76 | return report, err 77 | } 78 | 79 | return report, nil 80 | } 81 | 82 | func (a *api) Run(ctx context.Context, task metadata.Task) error { 83 | content, err := json.MarshalIndent(&task, "", " ") 84 | if err != nil { 85 | return err 86 | } 87 | 88 | req := a.client.NewRequest("POST", a.url("/run")). 89 | Body(bytes.NewReader(content)) 90 | 91 | resp, err := req.Send(ctx) 92 | if err != nil { 93 | return err 94 | } 95 | defer resp.Body.Close() 96 | 97 | logWriter := logutil.LogWriter(ctx) 98 | if logWriter != nil { 99 | err = logutil.WriteRemoteLogs(ctx, resp.Body, logWriter) 100 | if err != nil { 101 | return err 102 | } 103 | } 104 | 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /labapp/labapp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package labapp 16 | 17 | import ( 18 | "context" 19 | "io" 20 | 21 | "github.com/Netflix/p2plab/daemon" 22 | "github.com/Netflix/p2plab/labapp/approuter" 23 | "github.com/Netflix/p2plab/metadata" 24 | "github.com/Netflix/p2plab/peer" 25 | "github.com/rs/zerolog" 26 | ) 27 | 28 | type LabApp struct { 29 | daemon *daemon.Daemon 30 | peer *peer.Peer 31 | closers []io.Closer 32 | } 33 | 34 | func New(ctx context.Context, root, addr string, port int, logger *zerolog.Logger, pdef metadata.PeerDefinition) (*LabApp, error) { 35 | var closers []io.Closer 36 | pctx, cancel := context.WithCancel(ctx) 37 | p, err := peer.New(pctx, root, port, pdef) 38 | if err != nil { 39 | cancel() 40 | return nil, err 41 | } 42 | closers = append(closers, &daemon.CancelCloser{Cancel: cancel}) 43 | 44 | daemon, err := daemon.New("labapp", addr, logger, 45 | approuter.New(p), 46 | ) 47 | if err != nil { 48 | return nil, err 49 | } 50 | closers = append(closers, daemon) 51 | 52 | return &LabApp{ 53 | daemon: daemon, 54 | peer: p, 55 | closers: closers, 56 | }, nil 57 | } 58 | 59 | func (a *LabApp) Close() error { 60 | for _, closer := range a.closers { 61 | err := closer.Close() 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | return nil 67 | } 68 | 69 | func (a *LabApp) Serve(ctx context.Context) error { 70 | var addrs []string 71 | for _, ma := range a.peer.Host().Addrs() { 72 | addrs = append(addrs, ma.String()) 73 | } 74 | zerolog.Ctx(ctx).Info().Msgf("IPFS listening on %s", addrs) 75 | 76 | return a.daemon.Serve(ctx) 77 | } 78 | -------------------------------------------------------------------------------- /labd/controlapi/build.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controlapi 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "io" 21 | 22 | "github.com/Netflix/p2plab" 23 | "github.com/Netflix/p2plab/metadata" 24 | "github.com/Netflix/p2plab/pkg/httputil" 25 | ) 26 | 27 | type buildAPI struct { 28 | client *httputil.Client 29 | url urlFunc 30 | } 31 | 32 | func (a *buildAPI) Get(ctx context.Context, id string) (p2plab.Build, error) { 33 | req := a.client.NewRequest("GET", a.url("/builds/%s/json", id)) 34 | resp, err := req.Send(ctx) 35 | if err != nil { 36 | return nil, err 37 | } 38 | defer resp.Body.Close() 39 | 40 | var m metadata.Build 41 | err = json.NewDecoder(resp.Body).Decode(&m) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | return NewBuild(a.client, a.url, m), nil 47 | } 48 | 49 | func (a *buildAPI) List(ctx context.Context) ([]p2plab.Build, error) { 50 | req := a.client.NewRequest("GET", a.url("/builds/json")) 51 | 52 | resp, err := req.Send(ctx) 53 | if err != nil { 54 | return nil, err 55 | } 56 | defer resp.Body.Close() 57 | 58 | var metadatas []metadata.Build 59 | err = json.NewDecoder(resp.Body).Decode(&metadatas) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | var ns []p2plab.Build 65 | for _, m := range metadatas { 66 | ns = append(ns, NewBuild(a.client, a.url, m)) 67 | } 68 | 69 | return ns, nil 70 | } 71 | 72 | func (a *buildAPI) Upload(ctx context.Context, r io.Reader) (p2plab.Build, error) { 73 | req := a.client.NewRequest("POST", a.url("/builds/upload")). 74 | Body(r) 75 | 76 | resp, err := req.Send(ctx) 77 | if err != nil { 78 | return nil, err 79 | } 80 | defer resp.Body.Close() 81 | 82 | var m metadata.Build 83 | err = json.NewDecoder(resp.Body).Decode(&m) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return NewBuild(a.client, a.url, m), nil 89 | } 90 | 91 | type build struct { 92 | client *httputil.Client 93 | metadata metadata.Build 94 | url urlFunc 95 | } 96 | 97 | func NewBuild(client *httputil.Client, url urlFunc, m metadata.Build) p2plab.Build { 98 | return &build{ 99 | client: client, 100 | url: url, 101 | metadata: m, 102 | } 103 | } 104 | 105 | func (n *build) ID() string { 106 | return n.metadata.ID 107 | } 108 | 109 | func (n *build) Metadata() metadata.Build { 110 | return n.metadata 111 | } 112 | 113 | func (n *build) Open(ctx context.Context) (io.ReadCloser, error) { 114 | req := n.client.NewRequest("GET", n.url("/builds/%s/download", n.ID())) 115 | 116 | resp, err := req.Send(ctx) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | return resp.Body, nil 122 | } 123 | -------------------------------------------------------------------------------- /labd/controlapi/controlapi.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controlapi 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/Netflix/p2plab" 21 | "github.com/Netflix/p2plab/pkg/httputil" 22 | ) 23 | 24 | const ( 25 | ResourceID = "ResourceID" 26 | ) 27 | 28 | type api struct { 29 | addr string 30 | client *httputil.Client 31 | } 32 | 33 | func New(client *httputil.Client, addr string) p2plab.ControlAPI { 34 | return &api{ 35 | addr: addr, 36 | client: client, 37 | } 38 | } 39 | 40 | type urlFunc func(endpoint string, v ...interface{}) string 41 | 42 | func (a *api) url(endpoint string, v ...interface{}) string { 43 | return fmt.Sprintf("%s%s", a.addr, fmt.Sprintf(endpoint, v...)) 44 | } 45 | 46 | func (a *api) Cluster() p2plab.ClusterAPI { 47 | return &clusterAPI{a.client, a.url} 48 | } 49 | 50 | func (a *api) Node() p2plab.NodeAPI { 51 | return &nodeAPI{a.client, a.url} 52 | } 53 | 54 | func (a *api) Scenario() p2plab.ScenarioAPI { 55 | return &scenarioAPI{a.client, a.url} 56 | } 57 | 58 | func (a *api) Benchmark() p2plab.BenchmarkAPI { 59 | return &benchmarkAPI{a.client, a.url} 60 | } 61 | 62 | func (a *api) Experiment() p2plab.ExperimentAPI { 63 | return &experimentAPI{a.client, a.url} 64 | } 65 | 66 | func (a *api) Build() p2plab.BuildAPI { 67 | return &buildAPI{a.client, a.url} 68 | } 69 | -------------------------------------------------------------------------------- /labd/controlapi/node.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controlapi 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | "strings" 22 | 23 | "github.com/Netflix/p2plab" 24 | "github.com/Netflix/p2plab/labagent/agentapi" 25 | "github.com/Netflix/p2plab/labapp/appapi" 26 | "github.com/Netflix/p2plab/metadata" 27 | "github.com/Netflix/p2plab/pkg/httputil" 28 | ) 29 | 30 | type nodeAPI struct { 31 | client *httputil.Client 32 | url urlFunc 33 | } 34 | 35 | func (a *nodeAPI) Get(ctx context.Context, cluster, id string) (p2plab.Node, error) { 36 | req := a.client.NewRequest("GET", a.url("/clusters/%s/nodes/%s/json", cluster, id)) 37 | resp, err := req.Send(ctx) 38 | if err != nil { 39 | return nil, err 40 | } 41 | defer resp.Body.Close() 42 | 43 | var m metadata.Node 44 | err = json.NewDecoder(resp.Body).Decode(&m) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | return NewNode(a.client, m), nil 50 | } 51 | 52 | func (a *nodeAPI) Label(ctx context.Context, cluster string, ids, adds, removes []string) ([]p2plab.Node, error) { 53 | req := a.client.NewRequest("PUT", a.url("/clusters/%s/nodes/label", cluster)). 54 | Option("ids", strings.Join(ids, ",")) 55 | 56 | if len(adds) > 0 { 57 | req.Option("adds", strings.Join(adds, ",")) 58 | } 59 | if len(removes) > 0 { 60 | req.Option("removes", strings.Join(removes, ",")) 61 | } 62 | 63 | resp, err := req.Send(ctx) 64 | if err != nil { 65 | return nil, err 66 | } 67 | defer resp.Body.Close() 68 | 69 | var metadatas []metadata.Node 70 | err = json.NewDecoder(resp.Body).Decode(&metadatas) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | var nodes []p2plab.Node 76 | for _, m := range metadatas { 77 | nodes = append(nodes, NewNode(a.client, m)) 78 | } 79 | 80 | return nodes, nil 81 | } 82 | 83 | func (a *nodeAPI) List(ctx context.Context, cluster string, opts ...p2plab.ListOption) ([]p2plab.Node, error) { 84 | var settings p2plab.ListSettings 85 | for _, opt := range opts { 86 | err := opt(&settings) 87 | if err != nil { 88 | return nil, err 89 | } 90 | } 91 | 92 | req := a.client.NewRequest("GET", a.url("/clusters/%s/nodes/json", cluster)) 93 | if settings.Query != "" { 94 | req.Option("query", settings.Query) 95 | } 96 | 97 | resp, err := req.Send(ctx) 98 | if err != nil { 99 | return nil, err 100 | } 101 | defer resp.Body.Close() 102 | 103 | var metadatas []metadata.Node 104 | err = json.NewDecoder(resp.Body).Decode(&metadatas) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | var ns []p2plab.Node 110 | for _, m := range metadatas { 111 | ns = append(ns, NewNode(a.client, m)) 112 | } 113 | 114 | return ns, nil 115 | } 116 | 117 | type node struct { 118 | p2plab.AgentAPI 119 | p2plab.AppAPI 120 | metadata metadata.Node 121 | } 122 | 123 | func NewNode(client *httputil.Client, m metadata.Node) p2plab.Node { 124 | return &node{ 125 | AgentAPI: agentapi.New(client, fmt.Sprintf("http://%s:%d", m.Address, m.AgentPort)), 126 | AppAPI: appapi.New(client, fmt.Sprintf("http://%s:%d", m.Address, m.AppPort)), 127 | metadata: m, 128 | } 129 | } 130 | 131 | func (n *node) ID() string { 132 | return n.metadata.ID 133 | } 134 | 135 | func (n *node) Labels() []string { 136 | return n.metadata.Labels 137 | } 138 | 139 | func (n *node) Metadata() metadata.Node { 140 | return n.metadata 141 | } 142 | -------------------------------------------------------------------------------- /labd/routers/buildrouter/router.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package buildrouter 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "net/http" 21 | "net/url" 22 | 23 | "github.com/Netflix/p2plab" 24 | "github.com/Netflix/p2plab/daemon" 25 | "github.com/Netflix/p2plab/downloaders" 26 | "github.com/Netflix/p2plab/metadata" 27 | "github.com/google/uuid" 28 | ) 29 | 30 | type router struct { 31 | db metadata.DB 32 | uploader p2plab.Uploader 33 | fs *downloaders.Downloaders 34 | } 35 | 36 | func New(db metadata.DB, uploader p2plab.Uploader, fs *downloaders.Downloaders) daemon.Router { 37 | return &router{db, uploader, fs} 38 | } 39 | 40 | func (b *router) Routes() []daemon.Route { 41 | return []daemon.Route{ 42 | // GET 43 | daemon.NewGetRoute("/builds/json", b.getBuilds), 44 | daemon.NewGetRoute("/builds/{id}/json", b.getBuildByID), 45 | daemon.NewGetRoute("/builds/{id}/download", b.getBuildDownload), 46 | // POST 47 | daemon.NewPostRoute("/builds/upload", b.postBuildsUpload), 48 | } 49 | } 50 | 51 | func (b *router) getBuilds(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { 52 | builds, err := b.db.ListBuilds(ctx) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return daemon.WriteJSON(w, &builds) 58 | } 59 | 60 | func (b *router) getBuildByID(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { 61 | id := vars["id"] 62 | build, err := b.db.GetBuild(ctx, id) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | return daemon.WriteJSON(w, &build) 68 | } 69 | 70 | func (b *router) getBuildDownload(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { 71 | id := vars["id"] 72 | build, err := b.db.GetBuild(ctx, id) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | u, err := url.Parse(build.Link) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | downloader, err := b.fs.Get(u.Scheme) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | rc, err := downloader.Download(ctx, build.Link) 88 | if err != nil { 89 | return err 90 | } 91 | defer rc.Close() 92 | 93 | _, err = io.Copy(w, rc) 94 | return err 95 | } 96 | 97 | func (b *router) postBuildsUpload(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { 98 | link, err := b.uploader.Upload(ctx, r.Body) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | id, err := uuid.NewRandom() 104 | if err != nil { 105 | return err 106 | } 107 | 108 | build, err := b.db.CreateBuild(ctx, metadata.Build{ 109 | ID: id.String(), 110 | Link: link, 111 | }) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | return daemon.WriteJSON(w, &build) 117 | } 118 | -------------------------------------------------------------------------------- /labd/routers/helpers/doc.go: -------------------------------------------------------------------------------- 1 | // Package helpers contains helper functions to be reused by multiple routers 2 | package helpers 3 | -------------------------------------------------------------------------------- /labd/routers/helpers/helpers.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Netflix/p2plab" 7 | "github.com/Netflix/p2plab/labd/controlapi" 8 | "github.com/Netflix/p2plab/metadata" 9 | "github.com/Netflix/p2plab/nodes" 10 | "github.com/Netflix/p2plab/pkg/httputil" 11 | "github.com/pkg/errors" 12 | "github.com/rs/zerolog" 13 | bolt "go.etcd.io/bbolt" 14 | ) 15 | 16 | // Helper abstracts commonly used functions to be shared by any router 17 | type Helper struct { 18 | db metadata.DB 19 | provider p2plab.NodeProvider 20 | client *httputil.Client 21 | } 22 | 23 | // New instantiates our helper type 24 | func New(db metadata.DB, provider p2plab.NodeProvider, client *httputil.Client) *Helper { 25 | return &Helper{db, provider, client} 26 | } 27 | 28 | // CreateCluster enables creating the nodes in a cluster, waiting for them to be healthy before returning 29 | func (h *Helper) CreateCluster(ctx context.Context, cdef metadata.ClusterDefinition, name string) (metadata.Cluster, error) { 30 | var ( 31 | cluster = metadata.Cluster{ 32 | ID: name, 33 | Status: metadata.ClusterCreating, 34 | Definition: cdef, 35 | Labels: append([]string{ 36 | name, 37 | }, cdef.GenerateLabels()...), 38 | } 39 | err error 40 | ) 41 | 42 | cluster, err = h.db.CreateCluster(ctx, cluster) 43 | if err != nil { 44 | return cluster, err 45 | } 46 | 47 | zerolog.Ctx(ctx).Info().Str("cid", name).Msg("Creating node group") 48 | ng, err := h.provider.CreateNodeGroup(ctx, name, cdef) 49 | if err != nil { 50 | return cluster, err 51 | } 52 | 53 | var mns []metadata.Node 54 | cluster.Status = metadata.ClusterConnecting 55 | if err := h.db.Update(ctx, func(tx *bolt.Tx) error { 56 | var err error 57 | tctx := metadata.WithTransactionContext(ctx, tx) 58 | cluster, err = h.db.UpdateCluster(tctx, cluster) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | mns, err = h.db.CreateNodes(tctx, cluster.ID, ng.Nodes) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | return nil 69 | }); err != nil { 70 | return cluster, err 71 | } 72 | 73 | var ns = make([]p2plab.Node, len(mns)) 74 | for i, n := range mns { 75 | ns[i] = controlapi.NewNode(h.client, n) 76 | } 77 | 78 | if err := nodes.WaitHealthy(ctx, ns); err != nil { 79 | return cluster, err 80 | } 81 | 82 | cluster.Status = metadata.ClusterCreated 83 | return h.db.UpdateCluster(ctx, cluster) 84 | } 85 | 86 | func (h *Helper) DeleteCluster(ctx context.Context, name string) error { 87 | logger := zerolog.Ctx(ctx).With().Str("name", name).Logger() 88 | ctx = logger.WithContext(ctx) 89 | 90 | cluster, err := h.db.GetCluster(ctx, name) 91 | if err != nil { 92 | return errors.Wrapf(err, "failed to get cluster %q", name) 93 | } 94 | 95 | if cluster.Status != metadata.ClusterDestroying { 96 | cluster.Status = metadata.ClusterDestroying 97 | cluster, err = h.db.UpdateCluster(ctx, cluster) 98 | if err != nil { 99 | return errors.Wrap(err, "failed to update cluster status to destroying") 100 | } 101 | } 102 | 103 | ns, err := h.db.ListNodes(ctx, cluster.ID) 104 | if err != nil { 105 | return errors.Wrap(err, "failed to list nodes") 106 | } 107 | 108 | ng := &p2plab.NodeGroup{ 109 | ID: cluster.ID, 110 | Nodes: ns, 111 | } 112 | 113 | logger.Info().Msg("Destroying node group") 114 | err = h.provider.DestroyNodeGroup(ctx, ng) 115 | if err != nil { 116 | return errors.Wrap(err, "failed to destroy node group") 117 | } 118 | 119 | logger.Info().Msg("Deleting cluster metadata") 120 | err = h.db.DeleteCluster(ctx, cluster.ID) 121 | if err != nil { 122 | return errors.Wrap(err, "failed to delete cluster metadata") 123 | } 124 | 125 | logger.Info().Msg("Destroyed cluster") 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /labd/settings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package labd 16 | 17 | import ( 18 | "github.com/Netflix/p2plab/downloaders" 19 | "github.com/Netflix/p2plab/providers" 20 | "github.com/Netflix/p2plab/uploaders" 21 | ) 22 | 23 | type LabdOption func(*LabdSettings) error 24 | 25 | type LabdSettings struct { 26 | Libp2pPort int 27 | Provider string 28 | ProviderSettings providers.ProviderSettings 29 | Uploader string 30 | UploaderSettings uploaders.UploaderSettings 31 | DownloaderSettings downloaders.DownloaderSettings 32 | } 33 | 34 | func WithLibp2pPort(port int) LabdOption { 35 | return func(s *LabdSettings) error { 36 | s.Libp2pPort = port 37 | return nil 38 | } 39 | } 40 | 41 | func WithProvider(provider string) LabdOption { 42 | return func(s *LabdSettings) error { 43 | s.Provider = provider 44 | return nil 45 | } 46 | } 47 | 48 | func WithProviderSettings(settings providers.ProviderSettings) LabdOption { 49 | return func(s *LabdSettings) error { 50 | s.ProviderSettings = settings 51 | return nil 52 | } 53 | } 54 | 55 | func WithUploader(uploader string) LabdOption { 56 | return func(s *LabdSettings) error { 57 | s.Uploader = uploader 58 | return nil 59 | } 60 | } 61 | 62 | func WithUploaderSettings(settings uploaders.UploaderSettings) LabdOption { 63 | return func(s *LabdSettings) error { 64 | s.UploaderSettings = settings 65 | return nil 66 | } 67 | } 68 | 69 | func WithDownloaderSettings(settings downloaders.DownloaderSettings) LabdOption { 70 | return func(s *LabdSettings) error { 71 | s.DownloaderSettings = settings 72 | return nil 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /metadata/db_test.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/pkg/errors" 9 | bolt "go.etcd.io/bbolt" 10 | ) 11 | 12 | func TestDB(t *testing.T) { 13 | ctx, cancel := context.WithCancel(context.Background()) 14 | defer cancel() 15 | var testDir = "dbmetatest" 16 | db, cleanup := newTestDB(t, testDir) 17 | defer func() { 18 | if err := cleanup(); err != nil { 19 | t.Fatal(err) 20 | } 21 | }() 22 | var ( 23 | testBucket = "test" 24 | testKey = "testkey" 25 | testValue = "testvalue" 26 | ) 27 | // this should return an error as we have not populated the datastore 28 | if err := db.View(ctx, func(tx *bolt.Tx) error { 29 | bkt := tx.Bucket([]byte(testBucket)) 30 | if bkt == nil { 31 | return nil 32 | } 33 | return errors.New("found bucket") 34 | }); err != nil { 35 | t.Fatal(err) 36 | } 37 | if err := db.Update(ctx, func(tx *bolt.Tx) error { 38 | bkt, err := tx.CreateBucket([]byte(testBucket)) 39 | if err != nil { 40 | return err 41 | } 42 | return bkt.Put([]byte(testKey), []byte(testValue)) 43 | }); err != nil { 44 | t.Fatal(err) 45 | } 46 | if err := db.View(ctx, func(tx *bolt.Tx) error { 47 | bkt := tx.Bucket([]byte(testBucket)) 48 | if bkt == nil { 49 | return errors.New("should have found bucket") 50 | } 51 | data := bkt.Get([]byte(testKey)) 52 | if data == nil || string(data) != testValue { 53 | return errors.New("bad value ofund") 54 | } 55 | return nil 56 | }); err != nil { 57 | t.Fatal(err) 58 | } 59 | } 60 | 61 | func newTestDB(t *testing.T, path string) (DB, func() error) { 62 | db, err := NewDB(path) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | cleanup := func() error { 67 | if err := db.Close(); err != nil { 68 | return err 69 | } 70 | return os.RemoveAll(path) 71 | } 72 | return db, cleanup 73 | } 74 | -------------------------------------------------------------------------------- /metadata/report.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metadata 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "time" 21 | 22 | "github.com/Netflix/p2plab/errdefs" 23 | metrics "github.com/libp2p/go-libp2p-core/metrics" 24 | protocol "github.com/libp2p/go-libp2p-protocol" 25 | "github.com/pkg/errors" 26 | bolt "go.etcd.io/bbolt" 27 | ) 28 | 29 | type Report struct { 30 | Summary ReportSummary 31 | 32 | Aggregates ReportAggregates 33 | 34 | Nodes map[string]ReportNode 35 | 36 | Queries map[string][]string 37 | } 38 | 39 | type ReportSummary struct { 40 | TotalTime time.Duration 41 | 42 | Trace string 43 | 44 | Metrics string 45 | } 46 | 47 | type ReportAggregates struct { 48 | Totals ReportNode 49 | } 50 | 51 | type ReportNode struct { 52 | Bitswap ReportBitswap 53 | 54 | Bandwidth ReportBandwidth 55 | } 56 | 57 | type ReportBitswap struct { 58 | BlocksReceived uint64 59 | DataReceived uint64 60 | BlocksSent uint64 61 | DataSent uint64 62 | DupBlksReceived uint64 63 | DupDataReceived uint64 64 | MessagesReceived uint64 65 | } 66 | 67 | type ReportBandwidth struct { 68 | Totals metrics.Stats 69 | 70 | // TODO: Convert back to map[node id]. 71 | Peers map[string]metrics.Stats 72 | 73 | Protocols map[protocol.ID]metrics.Stats 74 | } 75 | 76 | func (m *db) GetReport(ctx context.Context, id string) (Report, error) { 77 | var report Report 78 | 79 | err := m.View(ctx, func(tx *bolt.Tx) error { 80 | bkt := getBenchmarksBucket(tx) 81 | if bkt == nil { 82 | return errors.Wrapf(errdefs.ErrNotFound, "benchmark %q", id) 83 | } 84 | 85 | bbkt := bkt.Bucket([]byte(id)) 86 | if bbkt == nil { 87 | return errors.Wrapf(errdefs.ErrNotFound, "benchmark %q", id) 88 | } 89 | 90 | err := readReport(bbkt, &report) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | return nil 96 | }) 97 | if err != nil { 98 | return report, err 99 | } 100 | 101 | return report, nil 102 | } 103 | 104 | func (m *db) CreateReport(ctx context.Context, id string, report Report) error { 105 | err := m.Update(ctx, func(tx *bolt.Tx) error { 106 | bkt, err := createBenchmarksBucket(tx) 107 | if err != nil { 108 | return err 109 | } 110 | 111 | bbkt := bkt.Bucket([]byte(id)) 112 | if bbkt == nil { 113 | return errors.Wrapf(errdefs.ErrNotFound, "benchmark %q", id) 114 | } 115 | 116 | return writeReport(bbkt, report) 117 | }) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | return nil 123 | } 124 | 125 | func readReport(bkt *bolt.Bucket, report *Report) error { 126 | content := bkt.Get(bucketKeyReport) 127 | if content == nil { 128 | return errors.Wrapf(errdefs.ErrNotFound, "no report available") 129 | } 130 | 131 | err := json.Unmarshal(content, report) 132 | if err != nil { 133 | return err 134 | } 135 | 136 | return nil 137 | } 138 | 139 | func writeReport(bkt *bolt.Bucket, report Report) error { 140 | content, err := json.Marshal(&report) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | err = bkt.Put(bucketKeyReport, content) 146 | if err != nil { 147 | return err 148 | } 149 | 150 | return nil 151 | } 152 | -------------------------------------------------------------------------------- /metadata/trial.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | // TrialDefinition is a grouping of a cluster, and scenario to run together 4 | type TrialDefinition struct { 5 | Cluster ClusterDefinition `json:"cluster"` 6 | Scenario ScenarioDefinition `json:"scenario"` 7 | } 8 | -------------------------------------------------------------------------------- /metadata/validate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metadata 16 | 17 | import ( 18 | "regexp" 19 | 20 | "github.com/Netflix/p2plab/errdefs" 21 | "github.com/pkg/errors" 22 | ) 23 | 24 | var ( 25 | ClusterIDPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,32}`) 26 | ) 27 | 28 | func ValidateClusterID(id string) error { 29 | match := ClusterIDPattern.MatchString(id) 30 | if !match { 31 | return errors.Wrapf(errdefs.ErrInvalidArgument, "cluster id must match %q", ClusterIDPattern) 32 | } 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /node.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | // NodeAPI defines the API for node operations. 24 | type NodeAPI interface { 25 | // Get returns a node. 26 | Get(ctx context.Context, cluster, id string) (Node, error) 27 | 28 | Label(ctx context.Context, cluster string, ids, adds, removes []string) ([]Node, error) 29 | 30 | List(ctx context.Context, cluster string, opts ...ListOption) ([]Node, error) 31 | } 32 | 33 | // Node is an instance running the P2P application to be benchmarked. 34 | type Node interface { 35 | Labeled 36 | 37 | AgentAPI 38 | 39 | AppAPI 40 | 41 | Metadata() metadata.Node 42 | } 43 | 44 | // NodeProvider is a service that can provision nodes. 45 | type NodeProvider interface { 46 | // CreateNodeGroup returns a healthy cluster of nodes. 47 | CreateNodeGroup(ctx context.Context, id string, cdef metadata.ClusterDefinition) (*NodeGroup, error) 48 | 49 | // DestroyNodeGroup destroys a cluster of nodes. 50 | DestroyNodeGroup(ctx context.Context, ng *NodeGroup) error 51 | } 52 | 53 | // NodeGroup is a cluster of nodes. 54 | type NodeGroup struct { 55 | ID string 56 | Nodes []metadata.Node 57 | } 58 | 59 | // SSHOption is an option to modify SSH settings. 60 | type SSHOption func(SSHSettings) error 61 | 62 | // SSHSetttings specify ssh settings when connecting to a node. 63 | type SSHSettings struct { 64 | } 65 | -------------------------------------------------------------------------------- /nodes/healthy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nodes 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/Netflix/p2plab" 22 | "github.com/Netflix/p2plab/errdefs" 23 | "github.com/Netflix/p2plab/pkg/logutil" 24 | "github.com/Netflix/p2plab/pkg/traceutil" 25 | "github.com/pkg/errors" 26 | "github.com/rs/zerolog" 27 | "golang.org/x/sync/errgroup" 28 | ) 29 | 30 | func WaitHealthy(ctx context.Context, ns []p2plab.Node) error { 31 | span, ctx := traceutil.StartSpanFromContext(ctx, "nodes.WaitHealthy") 32 | defer span.Finish() 33 | span.SetTag("nodes", len(ns)) 34 | 35 | healthchecks, gctx := errgroup.WithContext(ctx) 36 | 37 | zerolog.Ctx(ctx).Info().Msg("Waiting for healthy nodes") 38 | go logutil.Elapsed(gctx, 20*time.Second, "Waiting for healthy nodes") 39 | for _, n := range ns { 40 | n := n 41 | healthchecks.Go(func() error { 42 | ok := n.Healthcheck(gctx) 43 | if !ok { 44 | return errors.Wrapf(errdefs.ErrUnavailable, "node %q", n.ID()) 45 | } 46 | return nil 47 | }) 48 | } 49 | 50 | err := healthchecks.Wait() 51 | if err != nil { 52 | return err 53 | } 54 | 55 | return nil 56 | 57 | } 58 | -------------------------------------------------------------------------------- /nodes/reports.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nodes 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | "time" 21 | 22 | "github.com/Netflix/p2plab" 23 | "github.com/Netflix/p2plab/metadata" 24 | "github.com/Netflix/p2plab/pkg/logutil" 25 | "github.com/Netflix/p2plab/pkg/traceutil" 26 | "github.com/rs/zerolog" 27 | "golang.org/x/sync/errgroup" 28 | ) 29 | 30 | func CollectReports(ctx context.Context, ns []p2plab.Node) (map[string]metadata.ReportNode, error) { 31 | span, ctx := traceutil.StartSpanFromContext(ctx, "nodes.CollectReports") 32 | defer span.Finish() 33 | span.SetTag("nodes", len(ns)) 34 | 35 | getReports, gctx := errgroup.WithContext(ctx) 36 | 37 | zerolog.Ctx(ctx).Info().Msg("Retrieving reports") 38 | go logutil.Elapsed(gctx, 20*time.Second, "Retrieving reports") 39 | 40 | var mu sync.Mutex 41 | reportByNodeID := make(map[string]metadata.ReportNode) 42 | 43 | for _, n := range ns { 44 | n := n 45 | getReports.Go(func() error { 46 | report, err := n.Report(ctx) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | mu.Lock() 52 | reportByNodeID[n.ID()] = report 53 | mu.Unlock() 54 | return nil 55 | }) 56 | } 57 | 58 | err := getReports.Wait() 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return reportByNodeID, nil 64 | } 65 | -------------------------------------------------------------------------------- /nodes/session.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nodes 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/Netflix/p2plab" 22 | "github.com/Netflix/p2plab/pkg/logutil" 23 | "github.com/Netflix/p2plab/pkg/traceutil" 24 | opentracing "github.com/opentracing/opentracing-go" 25 | "github.com/rs/zerolog" 26 | "golang.org/x/sync/errgroup" 27 | ) 28 | 29 | func Session(ctx context.Context, ns []p2plab.Node, fn func(context.Context) error) (opentracing.Span, error) { 30 | span := traceutil.Tracer(ctx).StartSpan("scenarios.Session") 31 | defer span.Finish() 32 | sctx := opentracing.ContextWithSpan(ctx, span) 33 | 34 | var ids []string 35 | for _, n := range ns { 36 | ids = append(ids, n.ID()) 37 | } 38 | 39 | eg, gctx := errgroup.WithContext(sctx) 40 | 41 | zerolog.Ctx(ctx).Info().Msg("Starting a session for benchmarking") 42 | go logutil.Elapsed(gctx, 20*time.Second, "Starting a session for benchmarking") 43 | /* When running multiple different benchmarks using the same labd host 44 | the code below introduces a race condition, that will crash all running benchmarks. 45 | 46 | The issue likely happens due to the way context is shared between benchmarks and providers, 47 | but given that code below is only useful for data tracing, it has been left disabled. 48 | 49 | If looking into the bug, the following files and their packages may be worth a closer look: 50 | * labagent/supervisor/supervisor.go 51 | * labagent/agentrouter/router.go 52 | * labapp/appapi/appapi.go 53 | * labapp/approuter/router.go 54 | 55 | var cancels = make([]context.CancelFunc, 0, len(ns)) 56 | for _, n := range ns { 57 | n := n 58 | eg.Go(func() error { 59 | lctx, cancel := context.WithCancel(gctx) 60 | cancels = append(cancels, cancel) 61 | pdef := n.Metadata().Peer 62 | err := n.Update(lctx, n.ID(), "", pdef) 63 | if err != nil && !errdefs.IsCancelled(err) { 64 | return errors.Wrapf(err, "failed to update node %q", n.ID()) 65 | } 66 | 67 | return nil 68 | }) 69 | } 70 | */ 71 | err := WaitHealthy(ctx, ns) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | err = fn(sctx) 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | /*zerolog.Ctx(ctx).Info().Strs("nodes", ids).Msg("Ending the session") 82 | for _, cancel := range cancels { 83 | cancel() 84 | } 85 | */ 86 | err = eg.Wait() 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | return span, nil 92 | } 93 | -------------------------------------------------------------------------------- /peer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | "io" 20 | 21 | "github.com/Netflix/p2plab/metadata" 22 | cid "github.com/ipfs/go-cid" 23 | files "github.com/ipfs/go-ipfs-files" 24 | ipld "github.com/ipfs/go-ipld-format" 25 | host "github.com/libp2p/go-libp2p-core/host" 26 | "github.com/libp2p/go-libp2p-core/peer" 27 | ) 28 | 29 | // Peer is a minimal IPFS node that can distribute IPFS DAGs. 30 | type Peer interface { 31 | // Host returns the libp2p host. 32 | Host() host.Host 33 | 34 | // DAGService returns the IPLD DAG service. 35 | DAGService() ipld.DAGService 36 | 37 | // Connect connects to the libp2p peers. 38 | Connect(ctx context.Context, infos []peer.AddrInfo) error 39 | 40 | // Disconnect disconnects from libp2p peers. 41 | Disconnect(ctx context.Context, infos []peer.AddrInfo) error 42 | 43 | // Add adds content from an io.Reader into the Peer's storage. 44 | Add(ctx context.Context, r io.Reader, opts ...AddOption) (ipld.Node, error) 45 | 46 | // Get returns an Unixfsv1 file from a given cid. 47 | Get(ctx context.Context, c cid.Cid) (files.Node, error) 48 | 49 | // FetchGraph fetches the full DAG rooted at a given cid. 50 | FetchGraph(ctx context.Context, c cid.Cid) error 51 | 52 | // Report returns all the metrics collected from the peer. 53 | Report(ctx context.Context) (metadata.ReportNode, error) 54 | } 55 | 56 | // AddOption is an option for AddSettings. 57 | type AddOption func(*AddSettings) error 58 | 59 | // AddSettings describe the settings for adding content to the peer. 60 | type AddSettings struct { 61 | Layout string 62 | Chunker string 63 | RawLeaves bool 64 | Hidden bool 65 | Shard bool 66 | NoCopy bool 67 | HashFunc string 68 | MaxLinks int 69 | } 70 | 71 | // WithLayout sets the format for DAG generation. 72 | func WithLayout(layout string) AddOption { 73 | return func(s *AddSettings) error { 74 | s.Layout = layout 75 | return nil 76 | } 77 | } 78 | 79 | // WithChunker sets the chunking strategy for the content. 80 | func WithChunker(chunker string) AddOption { 81 | return func(s *AddSettings) error { 82 | s.Chunker = chunker 83 | return nil 84 | } 85 | } 86 | 87 | // WithRawLeaves sets whether to use raw blocks for leaf nodes. 88 | func WithRawLeaves(rawLeaves bool) AddOption { 89 | return func(s *AddSettings) error { 90 | s.RawLeaves = rawLeaves 91 | return nil 92 | } 93 | } 94 | 95 | // WithHashFunc sets the hashing function for the blocks. 96 | func WithHashFunc(hashFunc string) AddOption { 97 | return func(s *AddSettings) error { 98 | s.HashFunc = hashFunc 99 | return nil 100 | } 101 | } 102 | 103 | // WithMaxLinks sets the maximum children each block can have. 104 | func WithMaxLinks(maxLinks int) AddOption { 105 | return func(s *AddSettings) error { 106 | s.MaxLinks = maxLinks 107 | return nil 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /pkg/cliutil/attach.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cliutil 16 | 17 | import ( 18 | "context" 19 | "os" 20 | 21 | "github.com/rs/zerolog" 22 | "github.com/urfave/cli" 23 | ) 24 | 25 | func AttachAppContext(ctx context.Context, app *cli.App) { 26 | before := app.Before 27 | app.Before = func(c *cli.Context) error { 28 | if before != nil { 29 | if err := before(c); err != nil { 30 | return err 31 | } 32 | } 33 | 34 | level, err := zerolog.ParseLevel(c.GlobalString("log-level")) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | rootLogger := zerolog.New(os.Stderr).Level(level).With().Timestamp().Logger() 40 | logger := &rootLogger 41 | ctx = logger.WithContext(ctx) 42 | c.App.Metadata["context"] = ctx 43 | return nil 44 | } 45 | } 46 | 47 | func CommandContext(c *cli.Context) context.Context { 48 | return c.App.Metadata["context"].(context.Context) 49 | } 50 | 51 | func JoinBefore(fns ...cli.BeforeFunc) cli.BeforeFunc { 52 | return func(c *cli.Context) error { 53 | for _, fn := range fns { 54 | if fn == nil { 55 | continue 56 | } 57 | 58 | err := fn(c) 59 | if err != nil { 60 | return err 61 | } 62 | } 63 | return nil 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/cliutil/interrupt.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cliutil 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io" 21 | "os" 22 | "os/signal" 23 | "sync" 24 | "syscall" 25 | ) 26 | 27 | // InterruptHandler helps set up an interrupt handler that can be cleanly shut 28 | // down through the io.Closer interface. 29 | type InterruptHandler struct { 30 | sig chan os.Signal 31 | wg sync.WaitGroup 32 | } 33 | 34 | type handlerFunc func(ih *InterruptHandler, sig os.Signal) 35 | 36 | // NewInterruptHandler returns a new interrupt handler that will invoke cancel 37 | // if any of the signals provided are received. 38 | func NewInterruptHandler(cancel context.CancelFunc, sigs ...os.Signal) io.Closer { 39 | intrh := &InterruptHandler{ 40 | sig: make(chan os.Signal, 1), 41 | } 42 | 43 | count := 0 44 | handlerFunc := func(ih *InterruptHandler, sig os.Signal) { 45 | count++ 46 | switch count { 47 | case 1: 48 | // Prevent un-terminated ^C character in terminal. 49 | fmt.Println() 50 | 51 | fmt.Println("Gracefully cancelling request...") 52 | 53 | ih.wg.Add(1) 54 | go func() { 55 | defer ih.wg.Done() 56 | cancel() 57 | }() 58 | 59 | default: 60 | fmt.Println("Received another interrupt before graceful shutdown, terminating...") 61 | 62 | syscallSig, ok := sig.(syscall.Signal) 63 | if !ok { 64 | os.Exit(-1) 65 | } 66 | 67 | // Fatal errors exit with 128+n, where "n" is the syscall.Signal code. 68 | os.Exit(128 + int(syscallSig)) 69 | } 70 | } 71 | 72 | intrh.Handle(handlerFunc, sigs...) 73 | return intrh 74 | } 75 | 76 | // Close closes its signal receiver and waits for its handlers to exit cleanly. 77 | func (ih *InterruptHandler) Close() error { 78 | close(ih.sig) 79 | ih.wg.Wait() 80 | return nil 81 | } 82 | 83 | // Handle starts handling the given signals, and will call the handler callback 84 | // function each time a signal is catched. The function is passed the number of 85 | // times the handler has been triggered in total, as well as the handler itself, 86 | // so that the handling logic can use the handler's wait group to ensure clean 87 | // shutdown when Close() is called. 88 | func (ih *InterruptHandler) Handle(handler handlerFunc, sigs ...os.Signal) { 89 | signal.Notify(ih.sig, sigs...) 90 | 91 | ih.wg.Add(1) 92 | go func() { 93 | defer ih.wg.Done() 94 | for sig := range ih.sig { 95 | handler(ih, sig) 96 | } 97 | signal.Stop(ih.sig) 98 | }() 99 | } 100 | -------------------------------------------------------------------------------- /pkg/digestconv/digestconv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package digestconv 16 | 17 | import ( 18 | "encoding/hex" 19 | 20 | cid "github.com/ipfs/go-cid" 21 | multihash "github.com/multiformats/go-multihash" 22 | digest "github.com/opencontainers/go-digest" 23 | "github.com/pkg/errors" 24 | ) 25 | 26 | func DigestToCid(dgst digest.Digest) (cid.Cid, error) { 27 | // in exceptionally rare cases we may have an empty digest 28 | // this check here prevents a panic from happening in those cases 29 | if dgst.String() == "" { 30 | return cid.Cid{}, errors.New("bad digest") 31 | } 32 | data, err := hex.DecodeString(dgst.Hex()) 33 | if err != nil { 34 | return cid.Cid{}, errors.Wrap(err, "failed to decode digest hex") 35 | } 36 | 37 | encoded, err := multihash.Encode(data[:32], multihash.SHA2_256) 38 | if err != nil { 39 | return cid.Cid{}, errors.Wrap(err, "failed to encode digest as SHA256 multihash") 40 | } 41 | 42 | return cid.NewCidV1(cid.DagProtobuf, multihash.Multihash(encoded)), nil 43 | } 44 | 45 | func CidToDigest(c cid.Cid) (digest.Digest, error) { 46 | decoded, err := multihash.Decode(c.Hash()) 47 | if err != nil { 48 | return "", err 49 | } 50 | 51 | return digest.NewDigestFromBytes(digest.Canonical, decoded.Digest), nil 52 | } 53 | -------------------------------------------------------------------------------- /pkg/digestconv/digestconv_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package digestconv 16 | 17 | import ( 18 | "testing" 19 | 20 | cid "github.com/ipfs/go-cid" 21 | util "github.com/ipfs/go-ipfs-util" 22 | digest "github.com/opencontainers/go-digest" 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestDigestToCid(t *testing.T) { 27 | data := []byte("foobar") 28 | expected := cid.NewCidV1(cid.DagProtobuf, util.Hash(data)) 29 | actual, err := DigestToCid(digest.FromBytes(data)) 30 | require.NoError(t, err) 31 | require.Equal(t, expected.String(), actual.String()) 32 | } 33 | 34 | func TestCidToDigest(t *testing.T) { 35 | data := []byte("foobar") 36 | expected := digest.FromBytes(data) 37 | actual, err := CidToDigest(cid.NewCidV1(cid.DagProtobuf, util.Hash(data))) 38 | require.NoError(t, err) 39 | require.Equal(t, expected.String(), actual.String()) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/httputil/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package httputil 16 | 17 | import ( 18 | "net/http" 19 | "time" 20 | 21 | cleanhttp "github.com/hashicorp/go-cleanhttp" 22 | retryablehttp "github.com/hashicorp/go-retryablehttp" 23 | "github.com/opentracing-contrib/go-stdlib/nethttp" 24 | "github.com/rs/zerolog" 25 | ) 26 | 27 | func NewHTTPClient() *http.Client { 28 | return &http.Client{ 29 | Transport: &nethttp.Transport{ 30 | RoundTripper: cleanhttp.DefaultTransport(), 31 | }, 32 | } 33 | } 34 | 35 | type Client struct { 36 | HTTPClient *http.Client 37 | logger *zerolog.Logger 38 | } 39 | 40 | func NewClient(hclient *http.Client, opts ...ClientOption) (*Client, error) { 41 | client := &Client{ 42 | HTTPClient: hclient, 43 | } 44 | 45 | for _, opt := range opts { 46 | err := opt(client) 47 | if err != nil { 48 | return nil, err 49 | } 50 | } 51 | 52 | return client, nil 53 | } 54 | 55 | func (c *Client) NewRequest(method, url string, opts ...RequestOption) *Request { 56 | settings := RequestSettings{ 57 | RetryWaitMin: 1 * time.Second, 58 | RetryWaitMax: 30 * time.Second, 59 | RetryMax: 4, 60 | CheckRetry: retryablehttp.DefaultRetryPolicy, 61 | Backoff: retryablehttp.DefaultBackoff, 62 | } 63 | for _, opt := range opts { 64 | opt(&settings) 65 | } 66 | 67 | client := &retryablehttp.Client{ 68 | HTTPClient: c.HTTPClient, 69 | RetryWaitMin: settings.RetryWaitMin, 70 | RetryWaitMax: settings.RetryWaitMax, 71 | RetryMax: settings.RetryMax, 72 | CheckRetry: settings.CheckRetry, 73 | Backoff: settings.Backoff, 74 | } 75 | 76 | if c.logger != nil { 77 | client.Logger = c.logger 78 | } 79 | 80 | return &Request{ 81 | Method: method, 82 | Url: url, 83 | Options: make(map[string]string), 84 | client: client, 85 | } 86 | } 87 | 88 | type ClientOption func(*Client) error 89 | 90 | func WithLogger(logger *zerolog.Logger) ClientOption { 91 | return func(c *Client) error { 92 | c.logger = logger 93 | return nil 94 | } 95 | } 96 | 97 | type RequestOption func(*RequestSettings) 98 | 99 | type RequestSettings struct { 100 | RetryWaitMin time.Duration 101 | RetryWaitMax time.Duration 102 | RetryMax int 103 | CheckRetry retryablehttp.CheckRetry 104 | Backoff retryablehttp.Backoff 105 | } 106 | 107 | func WithRetryWaitMin(d time.Duration) RequestOption { 108 | return func(s *RequestSettings) { 109 | s.RetryWaitMin = d 110 | } 111 | } 112 | 113 | func WithRetryWaitMax(d time.Duration) RequestOption { 114 | return func(s *RequestSettings) { 115 | s.RetryWaitMax = d 116 | } 117 | } 118 | 119 | func WithRetryMax(max int) RequestOption { 120 | return func(s *RequestSettings) { 121 | s.RetryMax = max 122 | } 123 | } 124 | 125 | func WithCheckRetry(checkRetry retryablehttp.CheckRetry) RequestOption { 126 | return func(s *RequestSettings) { 127 | s.CheckRetry = checkRetry 128 | } 129 | } 130 | 131 | func WithBackoff(backoff retryablehttp.Backoff) RequestOption { 132 | return func(s *RequestSettings) { 133 | s.Backoff = backoff 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /pkg/httputil/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package httputil 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "net/http" 21 | ) 22 | 23 | type ErrorHandler struct { 24 | Handler func(w http.ResponseWriter, r *http.Request) error 25 | } 26 | 27 | type HTTPError interface { 28 | error 29 | Status() int 30 | } 31 | 32 | type StatusError struct { 33 | Code int 34 | Err error 35 | } 36 | 37 | func (se StatusError) Error() string { 38 | return se.Err.Error() 39 | } 40 | 41 | func (se StatusError) Status() int { 42 | return se.Code 43 | } 44 | 45 | // ServeHTTP allows our Handler type to satisfy http.Handler. 46 | func (h ErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 47 | err := h.Handler(w, r) 48 | if err != nil { 49 | switch e := err.(type) { 50 | case HTTPError: 51 | // We can retrieve the status here and write out a specific HTTP status code. 52 | fmt.Printf("HTTP %d - %s\n", e.Status(), e) 53 | http.Error(w, e.Error(), e.Status()) 54 | default: 55 | // Any error types we don't specifically look out for default to serving a 56 | // HTTP 500. 57 | http.Error(w, e.Error(), http.StatusInternalServerError) 58 | } 59 | } 60 | } 61 | 62 | func WriteJSON(w http.ResponseWriter, v interface{}) error { 63 | content, err := json.MarshalIndent(v, "", " ") 64 | if err != nil { 65 | return err 66 | } 67 | w.Write(content) 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /pkg/httputil/request.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package httputil 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "fmt" 21 | "io" 22 | "io/ioutil" 23 | "net/http" 24 | "net/url" 25 | "strconv" 26 | 27 | "github.com/Netflix/p2plab/errdefs" 28 | retryablehttp "github.com/hashicorp/go-retryablehttp" 29 | "github.com/opentracing-contrib/go-stdlib/nethttp" 30 | opentracing "github.com/opentracing/opentracing-go" 31 | "github.com/pkg/errors" 32 | "github.com/rs/zerolog/log" 33 | ) 34 | 35 | type Request struct { 36 | Method string 37 | Url string 38 | Options map[string]string 39 | body io.Reader 40 | 41 | client *retryablehttp.Client 42 | rawClient *http.Client 43 | } 44 | 45 | func (r *Request) Option(key string, value interface{}) *Request { 46 | var s string 47 | switch v := value.(type) { 48 | case bool: 49 | s = strconv.FormatBool(v) 50 | case string: 51 | s = v 52 | case []byte: 53 | s = string(v) 54 | default: 55 | s = fmt.Sprint(value) 56 | } 57 | 58 | r.Options[key] = s 59 | return r 60 | } 61 | 62 | func (r *Request) Body(value interface{}) *Request { 63 | var reader io.Reader 64 | switch v := value.(type) { 65 | case []byte: 66 | reader = bytes.NewReader(v) 67 | case string: 68 | reader = bytes.NewReader([]byte(v)) 69 | case io.Reader: 70 | reader = v 71 | } 72 | 73 | r.body = reader 74 | return r 75 | } 76 | 77 | func (r *Request) Send(ctx context.Context) (*http.Response, error) { 78 | u := r.url() 79 | req, err := http.NewRequest(r.Method, u, r.body) 80 | if err != nil { 81 | return nil, errors.Wrap(errdefs.ErrInvalidArgument, "failed to create new http request") 82 | } 83 | req = req.WithContext(ctx) 84 | 85 | span := opentracing.SpanFromContext(ctx) 86 | if span != nil { 87 | var ht *nethttp.Tracer 88 | req, ht = nethttp.TraceRequest(span.Tracer(), req) 89 | defer ht.Finish() 90 | } 91 | 92 | retryablereq, err := retryablehttp.FromRequest(req) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | resp, err := r.client.Do(retryablereq) 98 | if err != nil { 99 | return resp, errors.Wrap(err, "failed to do http request") 100 | } 101 | 102 | if (resp.StatusCode >= 400 && resp.StatusCode <= 499) || 103 | resp.StatusCode < 200 || resp.StatusCode > 299 { 104 | body, err := ioutil.ReadAll(resp.Body) 105 | if err != nil { 106 | log.Error().Msgf("failed to read body: %s", err) 107 | } 108 | defer resp.Body.Close() 109 | 110 | return nil, errors.Errorf("server rejected request [%d]: %s", resp.StatusCode, body) 111 | } 112 | 113 | return resp, nil 114 | } 115 | 116 | func (r *Request) url() string { 117 | values := make(url.Values) 118 | for k, v := range r.Options { 119 | values.Add(k, v) 120 | } 121 | return fmt.Sprintf("%s?%s", r.Url, values.Encode()) 122 | } 123 | -------------------------------------------------------------------------------- /pkg/logutil/console.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logutil 16 | 17 | import ( 18 | "context" 19 | "io" 20 | ) 21 | 22 | type writerKey struct{} 23 | 24 | func WithLogWriter(ctx context.Context, writer io.Writer) context.Context { 25 | return context.WithValue(ctx, writerKey{}, writer) 26 | } 27 | 28 | func LogWriter(ctx context.Context) io.Writer { 29 | value := ctx.Value(writerKey{}) 30 | writer, ok := value.(io.Writer) 31 | if !ok { 32 | return nil 33 | } 34 | return writer 35 | } 36 | -------------------------------------------------------------------------------- /pkg/logutil/elapsed.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logutil 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/hako/durafmt" 22 | "github.com/rs/zerolog" 23 | ) 24 | 25 | func Elapsed(ctx context.Context, tick time.Duration, msg string) { 26 | ticker := time.NewTicker(tick) 27 | defer ticker.Stop() 28 | 29 | var elapsed time.Duration 30 | for { 31 | select { 32 | case <-ctx.Done(): 33 | return 34 | case <-ticker.C: 35 | elapsed += tick 36 | zerolog.Ctx(ctx).Info().Str("elapsed", durafmt.Parse(elapsed).String()).Msg(msg) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/logutil/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logutil 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "net/http" 21 | "os" 22 | "sync" 23 | 24 | "github.com/rs/zerolog" 25 | ) 26 | 27 | func WithResponseLogger(ctx context.Context, w http.ResponseWriter) (context.Context, *zerolog.Logger) { 28 | multiwriter := io.MultiWriter(os.Stderr, NewWriteFlusher(w)) 29 | logger := zerolog.Ctx(ctx).Output(multiwriter) 30 | ctx = logger.WithContext(WithLogWriter(ctx, multiwriter)) 31 | return ctx, &logger 32 | } 33 | 34 | type WriteFlusher struct { 35 | w io.Writer 36 | f http.Flusher 37 | mu sync.Mutex 38 | } 39 | 40 | func NewWriteFlusher(w io.Writer) *WriteFlusher { 41 | wf := WriteFlusher{w: w} 42 | f, ok := w.(http.Flusher) 43 | if ok { 44 | wf.f = f 45 | } 46 | return &wf 47 | } 48 | 49 | func (wf *WriteFlusher) Write(p []byte) (int, error) { 50 | wf.mu.Lock() 51 | defer wf.mu.Unlock() 52 | 53 | n, err := wf.w.Write(p) 54 | if err != nil { 55 | return n, err 56 | } 57 | wf.f.Flush() 58 | return n, err 59 | } 60 | -------------------------------------------------------------------------------- /pkg/logutil/jaeger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logutil 16 | 17 | import ( 18 | "github.com/rs/zerolog" 19 | jaeger "github.com/uber/jaeger-client-go" 20 | ) 21 | 22 | type jaegerLogger struct { 23 | logger *zerolog.Logger 24 | } 25 | 26 | func NewJaegerLogger(logger *zerolog.Logger) jaeger.Logger { 27 | return &jaegerLogger{logger} 28 | } 29 | 30 | type Logger interface { 31 | // Error logs a message at error priority 32 | Error(msg string) 33 | 34 | // Infof logs a message at info priority 35 | Infof(msg string, args ...interface{}) 36 | } 37 | 38 | func (jl *jaegerLogger) Error(msg string) { 39 | jl.logger.Error().Msg(msg) 40 | } 41 | 42 | func (jl *jaegerLogger) Infof(msg string, args ...interface{}) { 43 | jl.logger.Info().Msgf(msg, args...) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/logutil/remote.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logutil 16 | 17 | import ( 18 | "bufio" 19 | "bytes" 20 | "context" 21 | "encoding/json" 22 | "io" 23 | 24 | "github.com/pkg/errors" 25 | "github.com/rs/zerolog" 26 | ) 27 | 28 | func WriteRemoteLogs(ctx context.Context, remote io.Reader, writer io.Writer) error { 29 | scanner := bufio.NewScanner(remote) 30 | for scanner.Scan() { 31 | decoder := json.NewDecoder(bytes.NewReader([]byte(scanner.Text()))) 32 | decoder.UseNumber() 33 | 34 | var evt map[string]interface{} 35 | err := decoder.Decode(&evt) 36 | if err != nil { 37 | zerolog.Ctx(ctx).Error().Msg(scanner.Text()) 38 | for scanner.Scan() { 39 | zerolog.Ctx(ctx).Error().Msg(scanner.Text()) 40 | } 41 | return errors.New("unexpected non-json response") 42 | } 43 | 44 | levelRaw, ok := evt[zerolog.LevelFieldName] 45 | if ok { 46 | levelStr, ok := levelRaw.(string) 47 | if ok { 48 | level, _ := zerolog.ParseLevel(levelStr) 49 | event := zerolog.Ctx(ctx).WithLevel(level) 50 | if event == nil { 51 | // If event is nil, then the log event was filtered out by the current 52 | // logger. 53 | continue 54 | } 55 | } 56 | } 57 | 58 | _, err = writer.Write(append(scanner.Bytes(), byte('\n'))) 59 | if err != nil { 60 | return err 61 | } 62 | } 63 | 64 | return scanner.Err() 65 | } 66 | -------------------------------------------------------------------------------- /pkg/logutil/writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logutil 16 | 17 | import ( 18 | "bufio" 19 | "io" 20 | 21 | "github.com/rs/zerolog" 22 | ) 23 | 24 | type logWriter struct { 25 | pipew io.WriteCloser 26 | scanner *bufio.Scanner 27 | } 28 | 29 | func NewWriter(logger *zerolog.Logger, level zerolog.Level) io.WriteCloser { 30 | piper, pipew := io.Pipe() 31 | scanner := bufio.NewScanner(piper) 32 | go func() { 33 | defer piper.Close() 34 | for scanner.Scan() { 35 | logger.WithLevel(level).Msg(scanner.Text()) 36 | } 37 | }() 38 | 39 | return &logWriter{pipew, scanner} 40 | } 41 | 42 | func (a *logWriter) Write(p []byte) (int, error) { 43 | return a.pipew.Write(p) 44 | } 45 | 46 | func (a *logWriter) Close() error { 47 | a.pipew.Close() 48 | return a.scanner.Err() 49 | } 50 | -------------------------------------------------------------------------------- /pkg/stringutil/stringutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package stringutil 16 | 17 | func Coalesce(slice []string) []string { 18 | var r []string 19 | for _, e := range slice { 20 | if e == "" { 21 | continue 22 | } 23 | r = append(r, e) 24 | } 25 | return r 26 | } 27 | -------------------------------------------------------------------------------- /pkg/traceutil/tracer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package traceutil 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "log" 21 | "os" 22 | "time" 23 | 24 | opentracing "github.com/opentracing/opentracing-go" 25 | jaeger "github.com/uber/jaeger-client-go" 26 | "github.com/uber/jaeger-client-go/config" 27 | ) 28 | 29 | type tracerKey struct{} 30 | 31 | func WithTracer(ctx context.Context, tracer opentracing.Tracer) context.Context { 32 | return context.WithValue(ctx, tracerKey{}, tracer) 33 | } 34 | 35 | func Tracer(ctx context.Context) opentracing.Tracer { 36 | tracer, ok := ctx.Value(tracerKey{}).(opentracing.Tracer) 37 | if !ok { 38 | return opentracing.NoopTracer{} 39 | } 40 | return tracer 41 | } 42 | 43 | func StartSpanFromContext(ctx context.Context, operationName string, opts ...opentracing.StartSpanOption) (opentracing.Span, context.Context) { 44 | return opentracing.StartSpanFromContextWithTracer(ctx, Tracer(ctx), operationName, opts...) 45 | } 46 | 47 | func New(ctx context.Context, service string, logger jaeger.Logger) (context.Context, opentracing.Tracer, io.Closer) { 48 | tracerAddr := os.Getenv("JAEGER_TRACE") 49 | if tracerAddr != "" { 50 | cfg := config.Configuration{ 51 | Sampler: &config.SamplerConfig{ 52 | Type: "const", 53 | Param: 1, 54 | }, 55 | Reporter: &config.ReporterConfig{ 56 | BufferFlushInterval: time.Second, 57 | LocalAgentHostPort: tracerAddr, 58 | }, 59 | } 60 | tracer, closer, err := cfg.New( 61 | service, 62 | config.Logger(logger), 63 | ) 64 | if err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | ctx = WithTracer(ctx, tracer) 69 | return ctx, tracer, closer 70 | } 71 | 72 | return ctx, opentracing.NoopTracer{}, &nopCloser{} 73 | } 74 | 75 | type nopCloser struct{} 76 | 77 | func (*nopCloser) Close() error { 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /printer/id.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package printer 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | type idPrinter struct{} 24 | 25 | func NewIDPrinter() Printer { 26 | return &idPrinter{} 27 | } 28 | 29 | func (p *idPrinter) Print(v interface{}) error { 30 | switch t := v.(type) { 31 | case []interface{}: 32 | for _, e := range t { 33 | err := p.Print(e) 34 | if err != nil { 35 | return err 36 | } 37 | } 38 | case metadata.Cluster: 39 | fmt.Printf("%s\n", t.ID) 40 | case metadata.Node: 41 | fmt.Printf("%s\n", t.ID) 42 | case metadata.Scenario: 43 | fmt.Printf("%s\n", t.ID) 44 | case metadata.Benchmark: 45 | fmt.Printf("%s\n", t.ID) 46 | case metadata.Experiment: 47 | fmt.Printf("%s\n", t.ID) 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /printer/json.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package printer 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | ) 21 | 22 | type jsonPrinter struct{} 23 | 24 | func NewJSONPrinter() Printer { 25 | return &jsonPrinter{} 26 | } 27 | 28 | func (p *jsonPrinter) Print(v interface{}) error { 29 | content, err := json.MarshalIndent(v, "", " ") 30 | if err != nil { 31 | return err 32 | } 33 | 34 | fmt.Println(string(content)) 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /printer/printer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package printer 16 | 17 | import ( 18 | "github.com/Netflix/p2plab/errdefs" 19 | "github.com/pkg/errors" 20 | ) 21 | 22 | type Printer interface { 23 | Print(v interface{}) error 24 | } 25 | 26 | type OutputType string 27 | 28 | var ( 29 | OutputAuto OutputType = "auto" 30 | OutputTable OutputType = "table" 31 | OutputID OutputType = "id" 32 | OutputUnix OutputType = "unix" 33 | OutputJSON OutputType = "json" 34 | ) 35 | 36 | func GetPrinter(output, auto OutputType) (Printer, error) { 37 | var p Printer 38 | switch output { 39 | case OutputAuto: 40 | if auto == OutputAuto { 41 | return nil, errors.Wrap(errdefs.ErrInvalidArgument, "auto printer cannot be auto") 42 | } 43 | return GetPrinter(auto, "") 44 | case OutputTable: 45 | p = NewTablePrinter() 46 | case OutputID: 47 | p = NewIDPrinter() 48 | case OutputUnix: 49 | p = NewUnixPrinter() 50 | case OutputJSON: 51 | p = NewJSONPrinter() 52 | default: 53 | return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "output %q is not valid", output) 54 | } 55 | return p, nil 56 | } 57 | -------------------------------------------------------------------------------- /printer/table.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package printer 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "strconv" 21 | "strings" 22 | 23 | "github.com/Netflix/p2plab/metadata" 24 | humanize "github.com/dustin/go-humanize" 25 | "github.com/olekukonko/tablewriter" 26 | ) 27 | 28 | type tablePrinter struct{} 29 | 30 | func NewTablePrinter() Printer { 31 | return &tablePrinter{} 32 | } 33 | 34 | func (p *tablePrinter) Print(v interface{}) error { 35 | table := tablewriter.NewWriter(os.Stdout) 36 | table.SetAutoFormatHeaders(false) 37 | 38 | switch t := v.(type) { 39 | case []interface{}: 40 | if len(t) > 0 { 41 | p.addHeader(table, t[0]) 42 | } else { 43 | fmt.Println("No results") 44 | return nil 45 | } 46 | for _, e := range t { 47 | p.addRow(table, e) 48 | } 49 | case metadata.Report: 50 | return printReport(t) 51 | default: 52 | p.addHeader(table, t) 53 | p.addRow(table, t) 54 | } 55 | 56 | table.Render() 57 | return nil 58 | } 59 | 60 | func (p *tablePrinter) addHeader(table *tablewriter.Table, v interface{}) { 61 | switch v.(type) { 62 | case metadata.Cluster: 63 | table.SetHeader([]string{"ID", "STATUS", "SIZE", "LABELS", "CREATEDAT", "UPDATEDAT"}) 64 | case metadata.Node: 65 | table.SetHeader([]string{"ID", "ADDRESS", "GITREFERENCE", "LABELS", "CREATEDAT", "UPDATEDAT"}) 66 | case metadata.Scenario: 67 | table.SetHeader([]string{"ID", "LABELS", "CREATEDAT", "UPDATEDAT"}) 68 | case metadata.Benchmark: 69 | table.SetHeader([]string{"ID", "STATUS", "CLUSTER", "SCENARIO", "LABELS", "CREATEDAT", "UPDATEDAT"}) 70 | case metadata.Experiment: 71 | table.SetHeader([]string{"ID", "STATUS", "LABELS", "CREATEDAT", "UPDATEDAT"}) 72 | case metadata.Build: 73 | table.SetHeader([]string{"ID", "LINK", "CREATEDAT", "UPDATEDAT"}) 74 | } 75 | } 76 | 77 | func (p *tablePrinter) addRow(table *tablewriter.Table, v interface{}) { 78 | switch t := v.(type) { 79 | case metadata.Cluster: 80 | table.Append([]string{ 81 | t.ID, 82 | string(t.Status), 83 | strconv.Itoa(t.Definition.Size()), 84 | strings.Join(t.Labels, ","), 85 | humanize.Time(t.CreatedAt), 86 | humanize.Time(t.UpdatedAt), 87 | }) 88 | case metadata.Node: 89 | table.Append([]string{ 90 | t.ID, 91 | t.Address, 92 | t.Peer.GitReference, 93 | strings.Join(t.Labels, ","), 94 | humanize.Time(t.CreatedAt), 95 | humanize.Time(t.UpdatedAt), 96 | }) 97 | case metadata.Scenario: 98 | table.Append([]string{ 99 | t.ID, 100 | strings.Join(t.Labels, ","), 101 | humanize.Time(t.CreatedAt), 102 | humanize.Time(t.UpdatedAt), 103 | }) 104 | case metadata.Benchmark: 105 | table.Append([]string{ 106 | t.ID, 107 | string(t.Status), 108 | t.Cluster.ID, 109 | t.Scenario.ID, 110 | strings.Join(t.Labels, ","), 111 | humanize.Time(t.CreatedAt), 112 | humanize.Time(t.UpdatedAt), 113 | }) 114 | case metadata.Experiment: 115 | table.Append([]string{ 116 | t.ID, 117 | string(t.Status), 118 | strings.Join(t.Labels, ","), 119 | humanize.Time(t.CreatedAt), 120 | humanize.Time(t.UpdatedAt), 121 | }) 122 | case metadata.Build: 123 | table.Append([]string{ 124 | t.ID, 125 | string(t.Link), 126 | humanize.Time(t.CreatedAt), 127 | humanize.Time(t.UpdatedAt), 128 | }) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /printer/unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package printer 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | type unixPrinter struct{} 24 | 25 | func NewUnixPrinter() Printer { 26 | return &unixPrinter{} 27 | } 28 | 29 | func (p *unixPrinter) Print(v interface{}) error { 30 | switch t := v.(type) { 31 | case []interface{}: 32 | for _, e := range t { 33 | err := p.Print(e) 34 | if err != nil { 35 | return err 36 | } 37 | } 38 | case metadata.Cluster: 39 | fmt.Printf("%s\n", t.ID) 40 | case metadata.Node: 41 | fmt.Printf("%s\n", t.ID) 42 | case metadata.Scenario: 43 | fmt.Printf("%s\n", t.ID) 44 | case metadata.Benchmark: 45 | fmt.Printf("%s\n", t.ID) 46 | case metadata.Experiment: 47 | fmt.Printf("%s\n", t.ID) 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /providers/providers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package providers 16 | 17 | import ( 18 | "path/filepath" 19 | 20 | "github.com/Netflix/p2plab" 21 | "github.com/Netflix/p2plab/downloaders" 22 | "github.com/Netflix/p2plab/downloaders/s3downloader" 23 | "github.com/Netflix/p2plab/errdefs" 24 | "github.com/Netflix/p2plab/labagent" 25 | "github.com/Netflix/p2plab/metadata" 26 | "github.com/Netflix/p2plab/providers/inmemory" 27 | "github.com/Netflix/p2plab/providers/terraform" 28 | "github.com/pkg/errors" 29 | "github.com/rs/zerolog" 30 | ) 31 | 32 | type ProviderSettings struct { 33 | DB metadata.DB 34 | Logger *zerolog.Logger 35 | } 36 | 37 | func GetNodeProvider(root, providerType string, settings ProviderSettings) (p2plab.NodeProvider, error) { 38 | root = filepath.Join(root, providerType) 39 | switch providerType { 40 | case "inmemory": 41 | return inmemory.New(root, settings.DB, settings.Logger, labagent.WithDownloaderSettings(downloaders.DownloaderSettings{ 42 | S3: s3downloader.S3DownloaderSettings{ 43 | Region: "us-west-2", 44 | }, 45 | }), 46 | ) 47 | case "terraform": 48 | return terraform.New(root) 49 | default: 50 | return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "unrecognized node provider type %q", providerType) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /providers/terraform/instances.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package terraform 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "encoding/json" 21 | "io" 22 | "os" 23 | "os/exec" 24 | ) 25 | 26 | type EC2Instance struct { 27 | InstanceId string `json:"InstanceId"` 28 | InstanceType string `json:"InstanceType"` 29 | PrivateIp string `json:"PrivateIpAddress"` 30 | } 31 | 32 | func DiscoverInstances(ctx context.Context, asg, region string) ([]EC2Instance, error) { 33 | asgStdout := new(bytes.Buffer) 34 | err := awscliWithStdio(ctx, asgStdout, nil, "autoscaling", "describe-auto-scaling-groups", 35 | "--query", "AutoScalingGroups[].Instances[].InstanceId", 36 | "--output", "json", 37 | "--region", region, 38 | "--auto-scaling-group-names", asg, 39 | ) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | var instanceIds []string 45 | err = json.NewDecoder(asgStdout).Decode(&instanceIds) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | instancesStdout := new(bytes.Buffer) 51 | err = awscliWithStdio(ctx, instancesStdout, nil, append([]string{"ec2", "describe-instances", 52 | "--query", "Reservations[].Instances[]", 53 | "--output", "json", 54 | "--region", region, 55 | "--instance-ids"}, instanceIds...)..., 56 | ) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | var instances []EC2Instance 62 | err = json.NewDecoder(instancesStdout).Decode(&instances) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return instances, nil 68 | } 69 | 70 | func awscli(ctx context.Context, args ...string) error { 71 | return awscliWithStdio(ctx, os.Stdout, os.Stderr, args...) 72 | } 73 | 74 | func awscliWithStdio(ctx context.Context, stdout, stderr io.Writer, args ...string) error { 75 | cmd := exec.CommandContext(ctx, "aws", args...) 76 | cmd.Stdin = nil 77 | cmd.Stdout = stdout 78 | cmd.Stderr = stderr 79 | return cmd.Run() 80 | } 81 | -------------------------------------------------------------------------------- /providers/terraform/templates/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.12.6" 3 | 4 | backend "s3" { 5 | bucket = "{{.Bucket}}" 6 | key = "{{.Key}}" 7 | region = "{{.Region}}" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | alias = "us-west-2" 13 | region = "us-west-2" 14 | } 15 | 16 | provider "aws" { 17 | alias = "us-east-1" 18 | region = "us-east-1" 19 | } 20 | 21 | provider "aws" { 22 | alias = "eu-west-1" 23 | region = "eu-west-1" 24 | } 25 | 26 | module "labagent_us-west-2" { 27 | source = "./modules/labagent" 28 | 29 | providers = { 30 | aws = aws.us-west-2 31 | } 32 | 33 | cluster_id = var.cluster_id 34 | labagents = var.labagents["us-west-2"] 35 | labagent_instance_profile = var.labagent_instance_profile 36 | internal_subnets = var.internal_subnets["us-west-2"] 37 | } 38 | 39 | module "labagent_us-east-1" { 40 | source = "./modules/labagent" 41 | 42 | providers = { 43 | aws = aws.us-east-1 44 | } 45 | 46 | cluster_id = var.cluster_id 47 | labagents = var.labagents["us-east-1"] 48 | labagent_instance_profile = var.labagent_instance_profile 49 | internal_subnets = var.internal_subnets["us-east-1"] 50 | } 51 | 52 | module "labagent_eu-west-1" { 53 | source = "./modules/labagent" 54 | 55 | providers = { 56 | aws = aws.eu-west-1 57 | } 58 | 59 | cluster_id = var.cluster_id 60 | labagents = var.labagents["eu-west-1"] 61 | labagent_instance_profile = var.labagent_instance_profile 62 | internal_subnets = var.internal_subnets["eu-west-1"] 63 | } 64 | -------------------------------------------------------------------------------- /providers/terraform/templates/modules/labagent/main.tf: -------------------------------------------------------------------------------- 1 | data "aws_ami" "labagent" { 2 | owners = ["self"] 3 | most_recent = true 4 | 5 | filter { 6 | name = "state" 7 | values = ["available"] 8 | } 9 | filter { 10 | name = "tag:Name" 11 | values = ["labagent"] 12 | } 13 | } 14 | 15 | data "aws_security_group" "labagent" { 16 | filter { 17 | name = "group-name" 18 | values = ["p2plab-labagent"] 19 | } 20 | } 21 | 22 | resource "aws_autoscaling_group" "labagent" { 23 | for_each = var.labagents 24 | 25 | name = each.key 26 | max_size = each.value.size 27 | min_size = each.value.size 28 | desired_capacity = each.value.size 29 | health_check_type = "EC2" 30 | vpc_zone_identifier = var.internal_subnets 31 | 32 | launch_template { 33 | id = aws_launch_template.labagent[each.key].id 34 | } 35 | } 36 | 37 | resource "aws_launch_template" "labagent" { 38 | for_each = var.labagents 39 | 40 | image_id = data.aws_ami.labagent.id 41 | name = each.key 42 | instance_type = each.value.instance_type 43 | vpc_security_group_ids = [data.aws_security_group.labagent.id] 44 | 45 | iam_instance_profile { 46 | name = var.labagent_instance_profile 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /providers/terraform/templates/modules/labagent/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /providers/terraform/templates/modules/labagent/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_id" { 2 | type = string 3 | } 4 | 5 | variable "labagent_instance_profile" { 6 | type = string 7 | } 8 | 9 | variable "labagents" { 10 | type = map(object({ 11 | size = number 12 | instance_type = string 13 | })) 14 | } 15 | 16 | variable "internal_subnets" { 17 | type = list(string) 18 | } 19 | -------------------------------------------------------------------------------- /providers/terraform/templates/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /providers/terraform/templates/terraform.tfvars: -------------------------------------------------------------------------------- 1 | cluster_id = "{{$.ID}}" 2 | 3 | labagents = { 4 | {{range .RegionalClusterGroups}} 5 | {{.Region}} = { 6 | {{range $i, $group := .Groups}} 7 | {{$.ID}}-{{$i}} = { 8 | size = {{$group.Size}} 9 | instance_type = "{{$group.InstanceType}}" 10 | } 11 | {{end}} 12 | } 13 | {{end}} 14 | } 15 | -------------------------------------------------------------------------------- /providers/terraform/templates/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_id" { 2 | type = string 3 | } 4 | 5 | variable "labagents" { 6 | type = map(map(object({ 7 | size = number 8 | instance_type = string 9 | }))) 10 | } 11 | 12 | variable "labagent_instance_profile" { 13 | default = "labagentInstanceProfile" 14 | } 15 | 16 | variable "internal_subnets" { 17 | default = { 18 | "us-west-2" = [ 19 | "subnet-090b3fe94bcc53cdb", 20 | "subnet-0148e2dfc8800fd39", 21 | "subnet-021adec36bd4bf0e2", 22 | ] 23 | "us-east-1" = [ 24 | "subnet-0a9b5ca758b008e3c", 25 | "subnet-052e37f4809f49e7d", 26 | "subnet-0bb444d1979bc6ce4", 27 | ] 28 | "eu-west-1" = [ 29 | "subnet-0ac98cc8ecb386361", 30 | "subnet-03339a637d558ac7f", 31 | "subnet-062c09248bdb6e857", 32 | ] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /query.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import "context" 18 | 19 | // Labeled defines a resource that has labels. 20 | type Labeled interface { 21 | // ID returns a uniquely identifiable string. 22 | ID() string 23 | 24 | // Labels returns a unique list of labels. 25 | Labels() []string 26 | } 27 | 28 | // LabeledSet is a set of labeled resources, duplicate resources are detected 29 | // by the ID of the labeled resource. 30 | type LabeledSet interface { 31 | // Add adds a labeled resource to the set. 32 | Add(labeled Labeled) 33 | 34 | // Remove removes a labeled resource from the set. 35 | Remove(id string) 36 | 37 | // Get returns a labeled resource from the set. 38 | Get(id string) Labeled 39 | 40 | // Contains returns whether a labeled resource with the id exists in the set. 41 | Contains(id string) bool 42 | 43 | // Slice returns the labeled resources as a slice. 44 | Slice() []Labeled 45 | } 46 | 47 | // Query is an executable function against a cluster to match a set of nodes. 48 | // Queries are used to group nodes to perform actions in either the seeding or 49 | // benchmarking stage of a scenario. 50 | type Query interface { 51 | // String returns the original query. 52 | String() string 53 | 54 | // Match returns the subset of lset that matches the query. 55 | Match(ctx context.Context, lset LabeledSet) (LabeledSet, error) 56 | } 57 | -------------------------------------------------------------------------------- /query/execute.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package query 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab" 21 | ) 22 | 23 | func Execute(ctx context.Context, ls []p2plab.Labeled, q string) (p2plab.LabeledSet, error) { 24 | qry, err := Parse(ctx, q) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | lset := NewLabeledSet() 30 | for _, l := range ls { 31 | lset.Add(l) 32 | } 33 | 34 | mset, err := qry.Match(ctx, lset) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | return mset, nil 40 | } 41 | -------------------------------------------------------------------------------- /query/execute_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package query 16 | 17 | import ( 18 | "testing" 19 | 20 | "context" 21 | 22 | "github.com/Netflix/p2plab" 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | var ls = []p2plab.Labeled{ 27 | NewLabeled("apple", []string{"everyone", "apple", "slowdisk", "region=us-west-2"}), 28 | NewLabeled("banana", []string{"everyone", "banana", "region=us-west-2"}), 29 | NewLabeled("cherry", []string{"everyone", "cherry", "region=us-east-1"}), 30 | } 31 | 32 | var executetest = []struct { 33 | in string 34 | out []p2plab.Labeled 35 | }{ 36 | {"'apple'", []p2plab.Labeled{ls[0]}}, 37 | {"(not 'apple')", []p2plab.Labeled{ls[1], ls[2]}}, 38 | {"(and 'slowdisk' 'region=us-west-2')", []p2plab.Labeled{ls[0]}}, 39 | {"(or 'region=us-west-2' 'region=us-east-1')", ls}, 40 | {"(or (not 'slowdisk') 'banana')", []p2plab.Labeled{ls[1], ls[2]}}, 41 | } 42 | 43 | func TestExecute(t *testing.T) { 44 | ctx := context.Background() 45 | 46 | for _, execute := range executetest { 47 | labeledSet, err := Execute(ctx, ls, execute.in) 48 | 49 | require.NoError(t, err) 50 | require.Equal(t, execute.out, labeledSet.Slice()) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /query/labeled.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package query 16 | 17 | import "github.com/Netflix/p2plab" 18 | 19 | type labeled struct { 20 | id string 21 | labels []string 22 | } 23 | 24 | func NewLabeled(id string, labels []string) p2plab.Labeled { 25 | return &labeled{id, labels} 26 | } 27 | 28 | func (l *labeled) ID() string { 29 | return l.id 30 | } 31 | 32 | func (l *labeled) Labels() []string { 33 | return l.labels 34 | } 35 | -------------------------------------------------------------------------------- /query/labeled_set.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package query 16 | 17 | import ( 18 | "sort" 19 | 20 | "github.com/Netflix/p2plab" 21 | ) 22 | 23 | type labeledSet struct { 24 | set map[string]p2plab.Labeled 25 | } 26 | 27 | func NewLabeledSet() p2plab.LabeledSet { 28 | return &labeledSet{ 29 | set: make(map[string]p2plab.Labeled), 30 | } 31 | } 32 | 33 | func (s *labeledSet) Add(l p2plab.Labeled) { 34 | s.set[l.ID()] = l 35 | } 36 | 37 | func (s *labeledSet) Remove(id string) { 38 | delete(s.set, id) 39 | } 40 | 41 | func (s *labeledSet) Get(id string) p2plab.Labeled { 42 | return s.set[id] 43 | } 44 | 45 | func (s *labeledSet) Contains(id string) bool { 46 | return s.Get(id) != nil 47 | } 48 | 49 | func (s *labeledSet) Slice() []p2plab.Labeled { 50 | var slice []p2plab.Labeled 51 | for _, l := range s.set { 52 | slice = append(slice, l) 53 | } 54 | sort.SliceStable(slice, func(i, j int) bool { 55 | return slice[i].ID() < slice[j].ID() 56 | }) 57 | return slice 58 | } 59 | -------------------------------------------------------------------------------- /query/labeled_set_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package query 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/Netflix/p2plab" 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | type testLabeled struct { 25 | id string 26 | } 27 | 28 | func (l *testLabeled) ID() string { 29 | return l.id 30 | } 31 | 32 | func (l *testLabeled) Labels() []string { 33 | return nil 34 | } 35 | 36 | func newLabeled(id string) p2plab.Labeled { 37 | return &testLabeled{id} 38 | } 39 | 40 | func TestLabeledSetAdd(t *testing.T) { 41 | // Empty set starts with 0 elements. 42 | lset := NewLabeledSet() 43 | require.Empty(t, lset.Slice()) 44 | 45 | // Adding node increases length by 1. 46 | lset.Add(newLabeled("1")) 47 | require.Len(t, lset.Slice(), 1) 48 | 49 | // Adding duplicate node does nothing. 50 | lset.Add(newLabeled("1")) 51 | require.Len(t, lset.Slice(), 1) 52 | } 53 | 54 | func TestLabeledSetRemove(t *testing.T) { 55 | lset := NewLabeledSet() 56 | lset.Add(newLabeled("1")) 57 | require.Len(t, lset.Slice(), 1) 58 | 59 | // Removing non-existing node does nothing. 60 | lset.Remove("2") 61 | require.Len(t, lset.Slice(), 1) 62 | 63 | // Removing existing node decreases length by 1. 64 | lset.Remove("1") 65 | require.Empty(t, lset.Slice()) 66 | 67 | // Removing pre-existing node does nothing. 68 | lset.Remove("1") 69 | require.Empty(t, lset.Slice()) 70 | } 71 | -------------------------------------------------------------------------------- /reports/aggregates.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package reports 16 | 17 | import "github.com/Netflix/p2plab/metadata" 18 | 19 | type uint64Pair struct { 20 | single uint64 21 | aggregate *uint64 22 | } 23 | 24 | type int64Pair struct { 25 | single int64 26 | aggregate *int64 27 | } 28 | 29 | type float64Pair struct { 30 | single float64 31 | aggregate *float64 32 | } 33 | 34 | func ComputeAggregates(reportByNodeId map[string]metadata.ReportNode) metadata.ReportAggregates { 35 | var aggregates metadata.ReportAggregates 36 | for _, reportNode := range reportByNodeId { 37 | bswap := reportNode.Bitswap 38 | 39 | for _, pair := range []uint64Pair{ 40 | {bswap.BlocksReceived, &aggregates.Totals.Bitswap.BlocksReceived}, 41 | {bswap.DataReceived, &aggregates.Totals.Bitswap.DataReceived}, 42 | {bswap.BlocksSent, &aggregates.Totals.Bitswap.BlocksSent}, 43 | {bswap.DataSent, &aggregates.Totals.Bitswap.DataSent}, 44 | {bswap.DupBlksReceived, &aggregates.Totals.Bitswap.DupBlksReceived}, 45 | {bswap.DupDataReceived, &aggregates.Totals.Bitswap.DupDataReceived}, 46 | {bswap.MessagesReceived, &aggregates.Totals.Bitswap.MessagesReceived}, 47 | } { 48 | *pair.aggregate += pair.single 49 | } 50 | 51 | bandwidth := reportNode.Bandwidth.Totals 52 | for _, pair := range []int64Pair{ 53 | {bandwidth.TotalIn, &aggregates.Totals.Bandwidth.Totals.TotalIn}, 54 | {bandwidth.TotalOut, &aggregates.Totals.Bandwidth.Totals.TotalOut}, 55 | } { 56 | *pair.aggregate += pair.single 57 | } 58 | 59 | for _, pair := range []float64Pair{ 60 | {bandwidth.RateIn, &aggregates.Totals.Bandwidth.Totals.RateIn}, 61 | {bandwidth.RateOut, &aggregates.Totals.Bandwidth.Totals.RateOut}, 62 | } { 63 | *pair.aggregate += pair.single 64 | } 65 | } 66 | return aggregates 67 | } 68 | -------------------------------------------------------------------------------- /scenario.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/Netflix/p2plab/metadata" 21 | ) 22 | 23 | // ScenarioAPI defines API for scenario operations. 24 | type ScenarioAPI interface { 25 | // Create saves a scenario for the given scenario definition. 26 | Create(ctx context.Context, name string, sdef metadata.ScenarioDefinition) (Scenario, error) 27 | 28 | // Get returns a scenario. 29 | Get(ctx context.Context, name string) (Scenario, error) 30 | 31 | // Label adds and removes labels from nodes identified by the list of names. 32 | Label(ctx context.Context, names, adds, removes []string) ([]Scenario, error) 33 | 34 | // List returns available scenarios. 35 | List(ctx context.Context, opts ...ListOption) ([]Scenario, error) 36 | 37 | Remove(ctx context.Context, names ...string) error 38 | } 39 | 40 | // Scenario is a schema for benchmarks that describes objects to benchmark, how 41 | // the cluster is initially seeded, and what to benchmark. 42 | type Scenario interface { 43 | Labeled 44 | 45 | Metadata() metadata.Scenario 46 | } 47 | -------------------------------------------------------------------------------- /scenarios/definition.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scenarios 16 | 17 | import ( 18 | "encoding/json" 19 | "io/ioutil" 20 | 21 | "github.com/Netflix/p2plab/metadata" 22 | ) 23 | 24 | func Parse(filename string) (metadata.ScenarioDefinition, error) { 25 | var sdef metadata.ScenarioDefinition 26 | content, err := ioutil.ReadFile(filename) 27 | if err != nil { 28 | return sdef, err 29 | } 30 | 31 | err = json.Unmarshal(content, &sdef) 32 | if err != nil { 33 | return sdef, err 34 | } 35 | 36 | return sdef, nil 37 | } 38 | -------------------------------------------------------------------------------- /transformer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package p2plab 16 | 17 | import ( 18 | "context" 19 | 20 | cid "github.com/ipfs/go-cid" 21 | ) 22 | 23 | // Transformer defines a way to convert an external resource into IPFS DAGs. 24 | type Transformer interface { 25 | // Transform adds a resource defined by source into an IPFS DAG stored in 26 | // peer. 27 | Transform(ctx context.Context, peer Peer, source string, opts ...AddOption) (cid.Cid, error) 28 | 29 | // Close releases any resources held by the Transformer. 30 | Close() error 31 | } 32 | -------------------------------------------------------------------------------- /transformers/oci/content.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package oci 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "io/ioutil" 21 | 22 | "github.com/Netflix/p2plab" 23 | "github.com/Netflix/p2plab/pkg/digestconv" 24 | "github.com/containerd/containerd/content" 25 | files "github.com/ipfs/go-ipfs-files" 26 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 27 | "github.com/pkg/errors" 28 | ) 29 | 30 | type provider struct { 31 | peer p2plab.Peer 32 | } 33 | 34 | func NewProvider(peer p2plab.Peer) content.Provider { 35 | return &provider{peer} 36 | } 37 | 38 | // ReaderAt only requires desc.Digest to be set. 39 | // Other fields in the descriptor may be used internally for resolving 40 | // the location of the actual data. 41 | func (p *provider) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { 42 | c, err := digestconv.DigestToCid(desc.Digest) 43 | if err != nil { 44 | return nil, errors.Wrapf(err, "failed to convert digest %q to cid", desc.Digest) 45 | } 46 | 47 | nd, err := p.peer.Get(ctx, c) 48 | if err != nil { 49 | return nil, errors.Wrapf(err, "failed to get file %q", c) 50 | } 51 | 52 | r := files.ToFile(nd) 53 | if r == nil { 54 | return nil, errors.New("expected node to be a unixfs file") 55 | } 56 | 57 | return &sizeReaderAt{ 58 | size: desc.Size, 59 | rc: r, 60 | }, nil 61 | } 62 | 63 | type sizeReaderAt struct { 64 | size int64 65 | rc io.ReadCloser 66 | n int64 67 | } 68 | 69 | func (ra *sizeReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { 70 | if offset < ra.n { 71 | return 0, errors.New("invalid offset") 72 | } 73 | diff := offset - ra.n 74 | written, err := io.CopyN(ioutil.Discard, ra.rc, diff) 75 | ra.n += written 76 | if err != nil { 77 | return int(written), err 78 | } 79 | 80 | n, err = ra.rc.Read(p) 81 | ra.n += int64(n) 82 | return 83 | } 84 | 85 | func (ra *sizeReaderAt) Size() int64 { 86 | return ra.size 87 | } 88 | 89 | func (ra *sizeReaderAt) Close() error { 90 | return ra.rc.Close() 91 | } 92 | -------------------------------------------------------------------------------- /transformers/oci/metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package oci 16 | 17 | import ( 18 | "encoding/binary" 19 | "fmt" 20 | 21 | "github.com/Netflix/p2plab/errdefs" 22 | digest "github.com/opencontainers/go-digest" 23 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 24 | bolt "go.etcd.io/bbolt" 25 | ) 26 | 27 | var ( 28 | bucketKeyDigest = []byte("digest") 29 | bucketKeyMediaType = []byte("mediaType") 30 | bucketKeySize = []byte("size") 31 | ) 32 | 33 | func (t *transformer) get(dgst digest.Digest) (desc ocispec.Descriptor, err error) { 34 | err = t.db.View(func(tx *bolt.Tx) error { 35 | bkt := tx.Bucket([]byte(dgst.String())) 36 | if bkt == nil { 37 | return errdefs.ErrNotFound 38 | } 39 | 40 | return bkt.ForEach(func(k, v []byte) error { 41 | if v == nil { 42 | return nil 43 | } 44 | 45 | switch string(k) { 46 | case string(bucketKeyDigest): 47 | desc.Digest = digest.Digest(v) 48 | case string(bucketKeyMediaType): 49 | desc.MediaType = string(v) 50 | case string(bucketKeySize): 51 | desc.Size, _ = binary.Varint(v) 52 | } 53 | 54 | return nil 55 | }) 56 | }) 57 | if err != nil { 58 | return desc, err 59 | } 60 | 61 | return desc, nil 62 | } 63 | 64 | func (t *transformer) put(dgst digest.Digest, desc ocispec.Descriptor) error { 65 | return t.db.Update(func(tx *bolt.Tx) error { 66 | bkt := tx.Bucket([]byte(dgst.String())) 67 | if bkt != nil { 68 | err := tx.DeleteBucket([]byte(dgst.String())) 69 | if err != nil { 70 | return err 71 | } 72 | } 73 | 74 | var err error 75 | bkt, err = tx.CreateBucket([]byte(dgst.String())) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | sizeEncoded, err := encodeInt(desc.Size) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | for _, v := range [][2][]byte{ 86 | {bucketKeyDigest, []byte(desc.Digest)}, 87 | {bucketKeyMediaType, []byte(desc.MediaType)}, 88 | {bucketKeySize, sizeEncoded}, 89 | } { 90 | if err := bkt.Put(v[0], v[1]); err != nil { 91 | return err 92 | } 93 | } 94 | 95 | return nil 96 | }) 97 | } 98 | 99 | func encodeInt(i int64) ([]byte, error) { 100 | var ( 101 | buf [binary.MaxVarintLen64]byte 102 | encoded = buf[:] 103 | ) 104 | encoded = encoded[:binary.PutVarint(encoded, i)] 105 | 106 | if len(encoded) == 0 { 107 | return nil, fmt.Errorf("failed encoding integer = %v", i) 108 | } 109 | return encoded, nil 110 | } 111 | -------------------------------------------------------------------------------- /transformers/transformers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package transformers 16 | 17 | import ( 18 | "net/http" 19 | "path/filepath" 20 | "sync" 21 | 22 | "github.com/Netflix/p2plab" 23 | "github.com/Netflix/p2plab/transformers/oci" 24 | "github.com/pkg/errors" 25 | ) 26 | 27 | type Transformers struct { 28 | root string 29 | client *http.Client 30 | mu sync.Mutex 31 | ts map[string]p2plab.Transformer 32 | } 33 | 34 | func New(root string, client *http.Client) *Transformers { 35 | return &Transformers{ 36 | root: root, 37 | client: client, 38 | ts: make(map[string]p2plab.Transformer), 39 | } 40 | } 41 | 42 | func (t *Transformers) Close() error { 43 | for _, transformer := range t.ts { 44 | err := transformer.Close() 45 | if err != nil { 46 | return err 47 | } 48 | } 49 | return nil 50 | } 51 | 52 | func (t *Transformers) Get(objectType string) (p2plab.Transformer, error) { 53 | t.mu.Lock() 54 | defer t.mu.Unlock() 55 | 56 | transformer, ok := t.ts[objectType] 57 | if !ok { 58 | var err error 59 | transformer, err = t.newTransformer(objectType) 60 | if err != nil { 61 | return nil, err 62 | } 63 | t.ts[objectType] = transformer 64 | } 65 | return transformer, nil 66 | } 67 | 68 | func (t *Transformers) newTransformer(objectType string) (p2plab.Transformer, error) { 69 | root := filepath.Join(t.root, objectType) 70 | switch objectType { 71 | case "oci": 72 | return oci.New(root, t.client) 73 | default: 74 | return nil, errors.Errorf("unrecognized object type: %q", objectType) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /uploaders/fileuploader/uploader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fileuploader 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "fmt" 21 | "io" 22 | "io/ioutil" 23 | "net/http" 24 | "os" 25 | "path/filepath" 26 | "time" 27 | 28 | "github.com/Netflix/p2plab" 29 | cid "github.com/ipfs/go-cid" 30 | multihash "github.com/multiformats/go-multihash" 31 | "github.com/rs/zerolog" 32 | ) 33 | 34 | type FileUploaderSettings struct { 35 | Address string 36 | } 37 | 38 | type uploader struct { 39 | root string 40 | cancel context.CancelFunc 41 | cidBuilder cid.Builder 42 | } 43 | 44 | func New(root string, logger *zerolog.Logger, settings FileUploaderSettings) (p2plab.Uploader, error) { 45 | root, err := filepath.Abs(root) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | err = os.MkdirAll(root, 0711) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | s := &http.Server{ 56 | Handler: http.FileServer(http.Dir(root)), 57 | Addr: settings.Address, 58 | ReadHeaderTimeout: 20 * time.Second, 59 | ReadTimeout: 1 * time.Minute, 60 | WriteTimeout: 30 * time.Minute, 61 | } 62 | 63 | ctx, cancel := context.WithCancel(context.Background()) 64 | go func() { 65 | <-ctx.Done() 66 | err := s.Shutdown(ctx) 67 | if err != nil { 68 | logger.Error().Err(err).Msg("failed to shutdown fsuploader") 69 | } 70 | }() 71 | 72 | go func() { 73 | err := s.ListenAndServe() 74 | if err != nil { 75 | logger.Error().Err(err).Msg("failed to serve fsuploader") 76 | } 77 | }() 78 | 79 | return &uploader{ 80 | root: root, 81 | cancel: cancel, 82 | cidBuilder: cid.V1Builder{MhType: multihash.SHA2_256}, 83 | }, nil 84 | } 85 | 86 | func (u *uploader) Close() error { 87 | u.cancel() 88 | return nil 89 | } 90 | 91 | func (u *uploader) Upload(ctx context.Context, r io.Reader) (link string, err error) { 92 | content, err := ioutil.ReadAll(r) 93 | if err != nil { 94 | return "", err 95 | } 96 | 97 | c, err := u.cidBuilder.Sum(content) 98 | if err != nil { 99 | return "", err 100 | } 101 | 102 | link = fmt.Sprintf("file://%s/%s", u.root, c) 103 | uploadPath := filepath.Join(u.root, c.String()) 104 | _, err = os.Stat(uploadPath) 105 | if err != nil && !os.IsNotExist(err) { 106 | return "", err 107 | } else if err == nil { 108 | return link, nil 109 | } 110 | 111 | f, err := os.Create(uploadPath) 112 | if err != nil { 113 | return "", err 114 | } 115 | defer f.Close() 116 | 117 | _, err = io.Copy(f, bytes.NewReader(content)) 118 | if err != nil { 119 | return "", err 120 | } 121 | 122 | return link, nil 123 | } 124 | -------------------------------------------------------------------------------- /uploaders/s3uploader/uploader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package s3uploader 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "fmt" 21 | "io" 22 | "io/ioutil" 23 | "net/http" 24 | "path" 25 | "time" 26 | 27 | "github.com/Netflix/p2plab" 28 | "github.com/Netflix/p2plab/pkg/logutil" 29 | "github.com/aws/aws-sdk-go-v2/aws" 30 | "github.com/aws/aws-sdk-go-v2/aws/external" 31 | "github.com/aws/aws-sdk-go-v2/service/s3/s3manager" 32 | cid "github.com/ipfs/go-cid" 33 | multihash "github.com/multiformats/go-multihash" 34 | "github.com/pkg/errors" 35 | "github.com/rs/zerolog" 36 | ) 37 | 38 | type S3UploaderSettings struct { 39 | Bucket string 40 | Prefix string 41 | Region string 42 | } 43 | 44 | type uploader struct { 45 | bucket string 46 | prefix string 47 | uploadManager *s3manager.Uploader 48 | cidBuilder cid.Builder 49 | } 50 | 51 | func New(client *http.Client, settings S3UploaderSettings) (p2plab.Uploader, error) { 52 | cfg, err := external.LoadDefaultAWSConfig() 53 | if err != nil { 54 | return nil, errors.Wrap(err, "failed to load aws config") 55 | } 56 | cfg.Region = settings.Region 57 | cfg.HTTPClient = client 58 | 59 | uploadManager := s3manager.NewUploader(cfg) 60 | return &uploader{ 61 | bucket: settings.Bucket, 62 | prefix: settings.Prefix, 63 | uploadManager: uploadManager, 64 | cidBuilder: cid.V1Builder{MhType: multihash.SHA2_256}, 65 | }, nil 66 | } 67 | 68 | func (u *uploader) Close() error { 69 | return nil 70 | } 71 | 72 | func (u *uploader) Upload(ctx context.Context, r io.Reader) (link string, err error) { 73 | content, err := ioutil.ReadAll(r) 74 | if err != nil { 75 | return "", err 76 | } 77 | 78 | c, err := u.cidBuilder.Sum(content) 79 | if err != nil { 80 | return "", err 81 | } 82 | 83 | key := path.Join(u.prefix, c.String()) 84 | logger := zerolog.Ctx(ctx).With().Str("bucket", u.bucket).Str("key", key).Logger() 85 | ectx, cancel := context.WithCancel(logger.WithContext(ctx)) 86 | defer cancel() 87 | 88 | go logutil.Elapsed(ectx, 20*time.Second, "Uploading S3 object") 89 | 90 | upParams := &s3manager.UploadInput{ 91 | Bucket: aws.String(u.bucket), 92 | Key: aws.String(key), 93 | Body: bytes.NewReader(content), 94 | } 95 | 96 | _, err = u.uploadManager.UploadWithContext(ctx, upParams) 97 | if err != nil { 98 | return "", err 99 | } 100 | 101 | return fmt.Sprintf("s3://%s/%s/%s", u.bucket, u.prefix, c), nil 102 | } 103 | -------------------------------------------------------------------------------- /uploaders/uploaders.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package uploaders 16 | 17 | import ( 18 | "path/filepath" 19 | 20 | "github.com/Netflix/p2plab" 21 | "github.com/Netflix/p2plab/errdefs" 22 | "github.com/Netflix/p2plab/pkg/httputil" 23 | "github.com/Netflix/p2plab/uploaders/fileuploader" 24 | "github.com/Netflix/p2plab/uploaders/s3uploader" 25 | "github.com/pkg/errors" 26 | "github.com/rs/zerolog" 27 | ) 28 | 29 | type UploaderSettings struct { 30 | Client *httputil.Client 31 | Logger *zerolog.Logger 32 | S3 s3uploader.S3UploaderSettings 33 | File fileuploader.FileUploaderSettings 34 | } 35 | 36 | func GetUploader(root, uploaderType string, settings UploaderSettings) (p2plab.Uploader, error) { 37 | root = filepath.Join(root, uploaderType) 38 | switch uploaderType { 39 | case "file": 40 | return fileuploader.New(root, settings.Logger, settings.File) 41 | case "s3": 42 | return s3uploader.New(settings.Client.HTTPClient, settings.S3) 43 | default: 44 | return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "unrecognized uploader type %q", uploaderType) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package version 16 | 17 | var ( 18 | // Package is filled at linking time 19 | Package = "github.com/Netflix/p2plab" 20 | 21 | // Version holds the complete version number. Filled in at linking time. 22 | Version = "0.0.1+unknown" 23 | 24 | // Revision is filled with the VCS (e.g. git) revision being used to build 25 | // the program at linking time. 26 | Revision = "" 27 | ) 28 | --------------------------------------------------------------------------------