├── pkg ├── sum │ ├── labeler │ │ ├── .gitignore │ │ ├── Dockerfile │ │ ├── main_test.go │ │ ├── labeler.go │ │ ├── labeler_test.go │ │ ├── labeler_e2e_test.go │ │ ├── main.go │ │ └── labeler_e2e_cmp_test.go │ ├── sumtestutil │ │ └── input.go │ ├── sum_concurrent.go │ ├── sum_test.go │ └── sum.go ├── memory │ ├── mmap │ │ ├── test_file.txt │ │ ├── mmap_test.go │ │ ├── mmap.go │ │ └── interactive │ │ │ ├── interactive_mmap.go │ │ │ └── interactive_open.go │ └── vars │ │ ├── vars.go │ │ └── vars_test.go ├── godoc │ ├── Makefile │ ├── .bingo │ │ ├── go.mod │ │ ├── bingo.mod │ │ ├── godoc.mod │ │ ├── .gitignore │ │ ├── variables.env │ │ ├── README.md │ │ ├── Variables.mk │ │ └── godoc.sum │ ├── block_test.go │ ├── README.md │ └── block.go ├── metrics │ ├── prom.yaml │ ├── cpu_test.go │ ├── dummy.go │ ├── httpmidleware │ │ └── middleware.go │ ├── mem_test.go │ └── latency_test.go ├── basic │ └── basic.go ├── testing │ ├── max.go │ ├── assert_test.go │ └── max_test.go ├── notationhungarian │ └── hung.go ├── unused │ └── unused.go ├── basicserver │ └── basicserver.go ├── leak │ ├── http_exhaust_test.go │ ├── file_test.go │ ├── http_exhaust.go │ ├── file.go │ ├── http_close.go │ └── http_close_test.go ├── profile │ └── fd │ │ ├── http.go │ │ ├── fd.go │ │ └── example │ │ └── main.go ├── compileroptimizeaway │ ├── opt_away.go │ └── opt_away_test.go ├── generics │ ├── sort.go │ ├── sort_test.go │ ├── blocks.go │ └── blocks_test.go ├── oop │ ├── oop_test.go │ └── oop.go ├── prealloc │ ├── slice.go │ ├── slice_test.go │ ├── linkedlist.go │ ├── prealloc_test.go │ └── linkedlist_test.go ├── concurrency │ ├── concurrency_test.go │ └── concurrency.go ├── pools │ ├── reuse.go │ └── reuse_test.go ├── export │ └── export.go ├── emptystruct │ ├── dups.go │ └── dups_test.go ├── getter │ ├── getter_test.go │ └── getter.go ├── errors │ └── errors.go └── json │ ├── json.go │ └── json_test.go ├── book.png ├── COPYRIGHT ├── .envrc ├── .bingo ├── go.mod ├── mdox.mod ├── golangci-lint.mod ├── .gitignore ├── copyright.mod ├── variables.env ├── README.md ├── Variables.mk └── copyright.sum ├── bw.bench.sh ├── .gitignore ├── .github └── workflows │ └── go.yaml ├── .golangci.yaml ├── Makefile ├── go.mod └── LICENSE /pkg/sum/labeler/.gitignore: -------------------------------------------------------------------------------- 1 | labeler 2 | 3 | e2e*/ -------------------------------------------------------------------------------- /book.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/efficientgo/examples/HEAD/book.png -------------------------------------------------------------------------------- /pkg/memory/mmap/test_file.txt: -------------------------------------------------------------------------------- 1 | This is a test string 2 | 3 | 4 | yolo. -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright (c) Efficient Go Authors 2 | Licensed under the Apache License 2.0. -------------------------------------------------------------------------------- /pkg/sum/labeler/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM busybox:1.35.0 2 | 3 | COPY labeler /labeler 4 | 5 | CMD ["/labeler"] -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | export GOPATH=~/Repos/sharedgopath 2 | export GOBIN=~/Repos/examples/.bin 3 | export PATH=${PATH}:${GOBIN} 4 | 5 | -------------------------------------------------------------------------------- /pkg/godoc/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | all: serve 3 | 4 | include .bingo/Variables.mk 5 | 6 | .PHONY: serve 7 | serve: $(GODOC) 8 | $(GODOC) -http=:6060 9 | -------------------------------------------------------------------------------- /pkg/metrics/prom.yaml: -------------------------------------------------------------------------------- 1 | scrape_configs: 2 | - job_name: "local" 3 | scrape_interval: "15s" 4 | static_configs: 5 | - targets: [ "localhost:8080" ] 6 | -------------------------------------------------------------------------------- /.bingo/go.mod: -------------------------------------------------------------------------------- 1 | module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. -------------------------------------------------------------------------------- /.bingo/mdox.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.21.4 4 | 5 | require github.com/bwplotka/mdox v0.9.0 6 | -------------------------------------------------------------------------------- /pkg/godoc/.bingo/go.mod: -------------------------------------------------------------------------------- 1 | module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. -------------------------------------------------------------------------------- /pkg/godoc/.bingo/bingo.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.16 4 | 5 | require github.com/bwplotka/bingo v0.4.3 6 | -------------------------------------------------------------------------------- /pkg/godoc/.bingo/godoc.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.16 4 | 5 | require golang.org/x/tools v0.1.2 // cmd/godoc 6 | -------------------------------------------------------------------------------- /.bingo/golangci-lint.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.20 4 | 5 | require github.com/golangci/golangci-lint v1.55.2 // cmd/golangci-lint 6 | -------------------------------------------------------------------------------- /.bingo/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Ignore everything 3 | * 4 | 5 | # But not these files: 6 | !.gitignore 7 | !*.mod 8 | !*.sum 9 | !README.md 10 | !Variables.mk 11 | !variables.env 12 | 13 | *tmp.mod 14 | -------------------------------------------------------------------------------- /pkg/godoc/.bingo/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Ignore everything 3 | * 4 | 5 | # But not these files: 6 | !.gitignore 7 | !*.mod 8 | !README.md 9 | !Variables.mk 10 | !variables.env 11 | 12 | *tmp.mod 13 | -------------------------------------------------------------------------------- /.bingo/copyright.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.17 4 | 5 | require github.com/efficientgo/tools/copyright v0.0.0-20210829154005-c7bad8450208 6 | -------------------------------------------------------------------------------- /pkg/basic/basic.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import "fmt" 7 | 8 | // Read more in "Efficient Go"; Example 2-1. 9 | func main() { 10 | fmt.Println("Hello World!") 11 | } 12 | -------------------------------------------------------------------------------- /bw.bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | set -xe 4 | 5 | export ver=v1 && \ 6 | go test ./pkg/json -run '^$' -bench '^BenchmarkLoad' -benchtime 10s -count 6 \ 7 | -cpu 4 \ 8 | -benchmem \ 9 | -memprofile=${ver}.mem.pprof -cpuprofile=${ver}.cpu.pprof \ 10 | | tee ${ver}.txt 11 | -------------------------------------------------------------------------------- /pkg/testing/max.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package max 5 | 6 | // max returns maximum of a and b. 7 | // If both are equal, returns a. 8 | // 9 | // Used as an example for unit tests. 10 | func max(a, b int) int { 11 | if a < b { 12 | return b 13 | } 14 | return a 15 | } 16 | -------------------------------------------------------------------------------- /pkg/testing/assert_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package max 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/efficientgo/core/testutil" 10 | ) 11 | 12 | func BenchmarkAssert(b *testing.B) { 13 | b.ReportAllocs() 14 | 15 | var err error 16 | 17 | b.ResetTimer() 18 | for i := 0; i < b.N; i++ { 19 | testutil.Ok(b, err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pkg/notationhungarian/hung.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package notationhungarian 5 | 6 | // structSystem represents old way of naming code structures (putting type in the name). It's anti-pattern nowadays. 7 | // Read more in "Efficient Go"; Example 1-5. 8 | type structSystem struct { 9 | sliceU32Numbers []uint32 10 | bCharacter byte 11 | f64Ratio float64 12 | } 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | *.idea 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | # vendor/ 17 | .bin 18 | 19 | *.pprof 20 | *.txt 21 | *.o 22 | *.a 23 | *.PKGDEF 24 | 25 | !results.txt 26 | !test_file.txt -------------------------------------------------------------------------------- /pkg/godoc/block_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package block_test 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | 10 | block "github.com/efficientgo/examples/pkg/godoc" 11 | "github.com/oklog/ulid" 12 | ) 13 | 14 | func ExampleDownload() { 15 | if err := block.Download(context.Background(), ulid.MustNew(0, nil), "here"); err != nil { 16 | fmt.Println(err) 17 | } 18 | // Output: downloaded 19 | } 20 | -------------------------------------------------------------------------------- /pkg/godoc/.bingo/variables.env: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.4.3. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | # Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk. 4 | GOBIN=${GOBIN:=$(go env GOBIN)} 5 | 6 | if [ -z "$GOBIN" ]; then 7 | GOBIN="$(go env GOPATH)/bin" 8 | fi 9 | 10 | 11 | BINGO="${GOBIN}/bingo-v0.4.3" 12 | 13 | GODOC="${GOBIN}/godoc-v0.1.2" 14 | 15 | -------------------------------------------------------------------------------- /pkg/unused/unused.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | // Example of compilation errors from unused variables. 7 | // Read more in "Efficient Go"; Example 2-7. 8 | 9 | func use(_ int) {} 10 | 11 | func main() { 12 | // var a int // error: a declared but not used 13 | 14 | // b := 1 // error: b declared but not used 15 | 16 | // var c int 17 | // d := c // error: d declared but not used 18 | 19 | e := 1 20 | use(e) 21 | 22 | f := 1 23 | _ = f 24 | } 25 | -------------------------------------------------------------------------------- /pkg/basicserver/basicserver.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "net/http" 8 | ) 9 | 10 | // Very minimal code for starting a web server. NOT production ready - always check errors and avoid globals (: 11 | // Read more in "Efficient Go"; Example 2-6. 12 | 13 | func handle(w http.ResponseWriter, _ *http.Request) { 14 | w.Write([]byte("It kind of works!")) 15 | } 16 | 17 | func main() { 18 | http.ListenAndServe(":8080", http.HandlerFunc(handle)) 19 | } 20 | -------------------------------------------------------------------------------- /.bingo/variables.env: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | # Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk. 4 | GOBIN=${GOBIN:=$(go env GOBIN)} 5 | 6 | if [ -z "$GOBIN" ]; then 7 | GOBIN="$(go env GOPATH)/bin" 8 | fi 9 | 10 | 11 | COPYRIGHT="${GOBIN}/copyright-v0.0.0-20210829154005-c7bad8450208" 12 | 13 | GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.55.2" 14 | 15 | MDOX="${GOBIN}/mdox-v0.9.0" 16 | 17 | -------------------------------------------------------------------------------- /pkg/memory/vars/vars.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package vars 5 | 6 | import ( 7 | "fmt" 8 | "unsafe" 9 | ) 10 | 11 | // Example showing differences between values pointers and special types used as function arguments. 12 | // Read more in "Efficient Go"; Example 5-4. 13 | 14 | func myFunction( 15 | arg1 int, arg2 *int, 16 | arg3 biggie, arg4 *biggie, 17 | arg5 []byte, arg6 *[]byte, 18 | arg7 chan byte, arg8 map[string]int, arg9 func(), 19 | ) { 20 | // ... 21 | fmt.Println(unsafe.Sizeof(arg3)) 22 | } 23 | 24 | type biggie struct { 25 | huge [1e8]byte 26 | other *biggie 27 | } 28 | -------------------------------------------------------------------------------- /pkg/leak/http_exhaust_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package leak 5 | 6 | import ( 7 | "net/http" 8 | "testing" 9 | 10 | "github.com/efficientgo/core/testutil" 11 | "go.uber.org/goleak" 12 | ) 13 | 14 | func BenchmarkClient(b *testing.B) { 15 | defer goleak.VerifyNone( 16 | b, 17 | goleak.IgnoreTopFunction("testing.(*B).run1"), 18 | goleak.IgnoreTopFunction("testing.(*B).doBench"), 19 | ) 20 | c := &http.Client{} 21 | defer c.CloseIdleConnections() 22 | b.ResetTimer() 23 | for i := 0; i < b.N; i++ { 24 | resp, err := c.Get("http://google.com") 25 | testutil.Ok(b, err) 26 | testutil.Ok(b, handleResp_Wrong(resp)) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/profile/fd/http.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package fd 5 | 6 | import ( 7 | "net/http" 8 | "net/http/pprof" 9 | 10 | "github.com/felixge/fgprof" 11 | ) 12 | 13 | // ExampleHTTP is an example of HTTP exposure of Go profiles. 14 | // Read more in "Efficient Go"; Example 9-5. 15 | func ExampleHTTP() { 16 | m := http.NewServeMux() 17 | m.HandleFunc("/debug/pprof/", pprof.Index) 18 | m.HandleFunc("/debug/pprof/profile", pprof.Profile) 19 | m.HandleFunc("/debug/fgprof/profile", fgprof.Handler().ServeHTTP) 20 | m.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 21 | 22 | srv := http.Server{Handler: m} 23 | 24 | // Start server... 25 | 26 | _ = srv 27 | } 28 | -------------------------------------------------------------------------------- /pkg/compileroptimizeaway/opt_away.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package compileroptimizeaway 5 | 6 | // Example of code that tends to be optimized by compiler in microbenchmarks. 7 | // Thanks to Dave Cheney for this example: https://dave.cheney.net/high-performance-go-workshop/gophercon-2019.html#watch_out_for_compiler_optimisations 8 | // Read more in "Efficient Go"; Example 8-16. 9 | 10 | const m1 = 0x5555555555555555 11 | const m2 = 0x3333333333333333 12 | const m4 = 0x0f0f0f0f0f0f0f0f 13 | const h01 = 0x0101010101010101 14 | 15 | func popcnt(x uint64) uint64 { 16 | x -= (x >> 1) & m1 17 | x = (x & m2) + ((x >> 2) & m2) 18 | x = (x + (x >> 4)) & m4 19 | return (x * h01) >> 56 20 | } 21 | -------------------------------------------------------------------------------- /pkg/godoc/README.md: -------------------------------------------------------------------------------- 1 | Example showcasing Go documentation with `godoc`. 2 | Read more in "Efficient Go"; Example 2-9. 3 | 4 | ## Usage 5 | 6 | Prerequisite: `go` installed. 7 | 8 | 1. Pull the repo on local machine. 9 | 2. Go to the directory where this README exists. 10 | 3. Run `make`. This will trigger `godoc` installation on your local machine. Then it will start `godoc` in serving mode that serves documentation on port 6060. 11 | 4. Once you see "using module mode; ...", go to `localhost:6060` in your browser. You see all available modules on your machine, including standard library. For `pkg/godoc` package see [http://localhost:6060/pkg/github.com/efficientgo/examples/pkg/godoc/](http://localhost:6060/pkg/github.com/efficientgo/examples/pkg/godoc/). 12 | -------------------------------------------------------------------------------- /pkg/generics/sort.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package generics 5 | 6 | import ( 7 | "sort" 8 | 9 | "golang.org/x/exp/constraints" 10 | ) 11 | 12 | // Example of generic sorting function for slice with any type that can be compared. 13 | // Read more in "Efficient Go"; Example 2-14. 14 | 15 | type genericSortableBasic[T constraints.Ordered] []T 16 | 17 | func (s genericSortableBasic[T]) Len() int { return len(s) } 18 | func (s genericSortableBasic[T]) Less(i, j int) bool { return s[i] < s[j] } 19 | func (s genericSortableBasic[T]) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 20 | 21 | func genericSortBasic[T constraints.Ordered](slice []T) { 22 | sort.Sort(genericSortableBasic[T](slice)) 23 | } 24 | -------------------------------------------------------------------------------- /pkg/oop/oop_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package block 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/efficientgo/core/testutil" 11 | "github.com/google/uuid" 12 | ) 13 | 14 | func TestCompactor(t *testing.T) { 15 | now := time.Now() 16 | 17 | block1 := Block{id: uuid.New(), start: now.Add(-2 * time.Hour), end: now} 18 | block2 := Block{id: uuid.New(), start: now.Add(-4 * time.Hour), end: now.Add(-2 * time.Hour)} 19 | 20 | compacted := Compact(block1, block2) 21 | testutil.Equals(t, 4*time.Hour, compacted.Duration()) 22 | testutil.Equals(t, 23 | compacted.id.String()+": "+block2.start.Format(time.RFC3339)+"-"+block1.end.Format(time.RFC3339), 24 | compacted.String(), 25 | ) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/sum/sumtestutil/input.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package sumtestutil 5 | 6 | import ( 7 | "io" 8 | 9 | "github.com/efficientgo/core/errors" 10 | ) 11 | 12 | func CreateTestInputWithExpectedResult(w io.Writer, numLen int) (sum int64, err error) { 13 | const testSumOfTen = int64(31108) 14 | var tenSet = []byte(`123 15 | 43 16 | 632 17 | 22 18 | 2 19 | 122 20 | 26660 21 | 91 22 | 2 23 | 3411 24 | `) 25 | 26 | if numLen%10 != 0 { 27 | return 0, errors.Newf("number of input should be division by 10, got %v", numLen) 28 | } 29 | 30 | for i := 0; i < numLen/10; i++ { 31 | if _, err := w.Write(tenSet); err != nil { 32 | return 0, err 33 | } 34 | } 35 | 36 | return testSumOfTen * (int64(numLen) / 10), nil 37 | } 38 | -------------------------------------------------------------------------------- /pkg/testing/max_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package max 5 | 6 | import ( 7 | "math" 8 | "testing" 9 | 10 | "github.com/efficientgo/core/testutil" 11 | ) 12 | 13 | // Read more in "Efficient Go"; Example 2-8. 14 | func TestMax(t *testing.T) { 15 | t.Parallel() 16 | 17 | for _, tcase := range []struct { 18 | a, b int 19 | expected int 20 | }{ 21 | {a: 0, b: 0, expected: 0}, 22 | {a: -1, b: 0, expected: 0}, 23 | {a: 1, b: 0, expected: 1}, 24 | {a: 0, b: -1, expected: 0}, 25 | {a: 0, b: 1, expected: 1}, 26 | {a: math.MinInt64, b: math.MaxInt64, expected: math.MaxInt64}, 27 | } { 28 | t.Run("", func(t *testing.T) { 29 | testutil.Equals(t, tcase.expected, max(tcase.a, tcase.b)) 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/leak/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package leak 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/efficientgo/core/testutil" 10 | "github.com/go-kit/log" 11 | ) 12 | 13 | func TestDoWithFile(t *testing.T) { 14 | testutil.Ok(t, doWithFile_Wrong("/dev/null")) 15 | testutil.Ok(t, doWithFile_CaptureCloseErr("/dev/null")) 16 | doWithFile_LogCloseErr(log.NewNopLogger(), "/dev/null") 17 | } 18 | 19 | func TestOpenMultiple(t *testing.T) { 20 | files, err := openMultiple_Wrong("/dev/null", "/dev/null", "/dev/null") 21 | testutil.Ok(t, err) 22 | testutil.Ok(t, closeAll(files)) 23 | 24 | files, err = openMultiple_Correct("/dev/null", "/dev/null", "/dev/null") 25 | testutil.Ok(t, err) 26 | testutil.Ok(t, closeAll(files)) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/prealloc/slice.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package prealloc 5 | 6 | // createSlice represents less efficient (also less readable IMO) creation of slice. 7 | // Read more in "Efficient Go"; Example 1-3. 8 | func createSlice(n int) (slice []string) { 9 | for i := 0; i < n; i++ { 10 | slice = append(slice, "I", "am", "going", "to", "take", "some", "space") 11 | } 12 | return slice 13 | } 14 | 15 | // createSlice_Better represents more efficient creation of slice that pre-allocates array in memory upfront. 16 | // Read more in "Efficient Go"; Example 1-4. 17 | func createSlice_Better(n int) []string { 18 | slice := make([]string, 0, n*7) 19 | for i := 0; i < n; i++ { 20 | slice = append(slice, "I", "am", "going", "to", "take", "some", "space") 21 | } 22 | return slice 23 | } 24 | -------------------------------------------------------------------------------- /.bingo/README.md: -------------------------------------------------------------------------------- 1 | # Project Development Dependencies. 2 | 3 | This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by https://github.com/bwplotka/bingo. 4 | 5 | * Run `bingo get` to install all tools having each own module file in this directory. 6 | * Run `bingo get ` to install that have own module file in this directory. 7 | * For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use $() variable where is the .bingo/.mod. 8 | * For shell: Run `source .bingo/variables.env` to source all environment variable for each tool. 9 | * For go: Import `.bingo/variables.go` to for variable names. 10 | * See https://github.com/bwplotka/bingo or -h on how to add, remove or change binaries dependencies. 11 | 12 | ## Requirements 13 | 14 | * Go 1.14+ 15 | -------------------------------------------------------------------------------- /pkg/concurrency/concurrency_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package concurrency 5 | 6 | import ( 7 | "sync" 8 | "testing" 9 | 10 | "github.com/efficientgo/core/testutil" 11 | ) 12 | 13 | func TestFunction(t *testing.T) { 14 | function() 15 | } 16 | 17 | func TestConcurrency(t *testing.T) { 18 | var mu sync.Mutex 19 | var num int64 20 | // Do not do that at home. Globals are bad, doing it so example is simpler (: 21 | randInt64 = func() int64 { 22 | mu.Lock() 23 | defer mu.Unlock() 24 | 25 | num += 10 26 | return num 27 | } 28 | 29 | testutil.Equals(t, int64(10+20+30), sharingWithAtomic()) 30 | testutil.Equals(t, int64(40+50+60), sharingWithMutex()) 31 | testutil.Equals(t, int64(70+80+90), sharingWithChannel()) 32 | testutil.Equals(t, int64(100+110+120), sharingWithShardedSpace()) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/metrics/cpu_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package metrics 5 | 6 | import ( 7 | "net/http" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/collectors" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | ) 13 | 14 | // Example of getting CPU usage through Prometheus metrics 15 | // Read more in "Efficient Go"; Example 6-11. 16 | func ExampleCPUTimeMetric() { 17 | reg := prometheus.NewRegistry() 18 | reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) 19 | 20 | go http.ListenAndServe(":8080", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) 21 | 22 | for i := 0; i < xTimes; i++ { 23 | err := doOperation() 24 | // ... 25 | _ = err 26 | } 27 | 28 | printPrometheusMetrics(reg) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/godoc/.bingo/README.md: -------------------------------------------------------------------------------- 1 | # Project Development Dependencies. 2 | 3 | This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by https://github.com/bwplotka/bingo. 4 | 5 | * Run `bingo get` to install all tools having each own module file in this directory. 6 | * Run `bingo get ` to install that have own module file in this directory. 7 | * For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use $() variable where is the .bingo/.mod. 8 | * For shell: Run `source .bingo/variables.env` to source all environment variable for each tool. 9 | * For go: Import `.bingo/variables.go` to for variable names. 10 | * See https://github.com/bwplotka/bingo or -h on how to add, remove or change binaries dependencies. 11 | 12 | ## Requirements 13 | 14 | * Go 1.14+ 15 | -------------------------------------------------------------------------------- /pkg/pools/reuse.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package pools 5 | 6 | import "sync" 7 | 8 | // Example of simple buffering and sync.Pool as a solution. 9 | // Read more in "Efficient Go"; Example 11-17, 11-18, 11-19. 10 | 11 | func processUsingBuffer(buf []byte) { 12 | buf = buf[:0] 13 | 14 | for i := 0; i < 1e6; i++ { 15 | buf = append(buf, 'a') 16 | } 17 | 18 | // Use buffer... 19 | } 20 | 21 | func processUsingPool_Wrong(p *sync.Pool) { 22 | buf := p.Get().([]byte) 23 | buf = buf[:0] 24 | 25 | defer p.Put(buf) 26 | 27 | for i := 0; i < 1e6; i++ { 28 | buf = append(buf, 'a') 29 | } 30 | 31 | // Use buffer... 32 | } 33 | 34 | func processUsingPool(p *sync.Pool) { 35 | buf := p.Get().([]byte) 36 | buf = buf[:0] 37 | 38 | for i := 0; i < 1e6; i++ { 39 | buf = append(buf, 'a') 40 | } 41 | defer p.Put(buf) 42 | 43 | // Use buffer... 44 | } 45 | -------------------------------------------------------------------------------- /pkg/export/export.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | // Examples of private and public constructs. 7 | // Read more in "Efficient Go"; Example 2-2. 8 | 9 | const privateConst = 1 10 | const PublicConst = 2 11 | 12 | var privateVar int 13 | var PublicVar int 14 | 15 | func privateFunc() {} 16 | func PublicFunc() {} 17 | 18 | type privateStruct struct { 19 | privateField int 20 | PublicField int 21 | } 22 | 23 | func (privateStruct) privateMethod() {} 24 | func (privateStruct) PublicMethod() {} 25 | 26 | type PublicStruct struct { 27 | privateField int 28 | PublicField int 29 | } 30 | 31 | func (PublicStruct) privateMethod() {} 32 | func (PublicStruct) PublicMethod() {} 33 | 34 | type privateInterface interface { 35 | privateMethod() 36 | PublicMethod() 37 | } 38 | 39 | type PublicInterface interface { 40 | privateMethod() 41 | PublicMethod() 42 | } 43 | -------------------------------------------------------------------------------- /pkg/generics/sort_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package generics 5 | 6 | import ( 7 | "sort" 8 | "testing" 9 | 10 | "github.com/efficientgo/core/testutil" 11 | ) 12 | 13 | func ExampleBasic() { 14 | toSort := []int{-20, 1, 10, 20} 15 | sort.Ints(toSort) 16 | 17 | toSort2 := []int{-20, 1, 10, 20} 18 | genericSortBasic[int](toSort2) 19 | 20 | // Output: 21 | } 22 | 23 | func TestSortSimple(t *testing.T) { 24 | expected := []int{-20, 1, 10, 20} 25 | unsorted := []int{10, 20, -20, 1} 26 | 27 | t.Run("sortable", func(t *testing.T) { 28 | toSort := make([]int, len(unsorted)) 29 | copy(toSort, unsorted) 30 | 31 | sort.Ints(toSort) 32 | 33 | testutil.Equals(t, expected, toSort) 34 | }) 35 | t.Run("generics", func(t *testing.T) { 36 | toSort := make([]int, len(unsorted)) 37 | copy(toSort, unsorted) 38 | 39 | genericSortBasic[int](toSort) 40 | 41 | testutil.Equals(t, expected, toSort) 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/prealloc/slice_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package prealloc 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/efficientgo/core/testutil" 10 | ) 11 | 12 | func TestCreateSlice(t *testing.T) { 13 | const n = int(1e4) 14 | 15 | wp := createSlice(n) 16 | testutil.Equals(t, n*7, len(wp)) 17 | 18 | // We can't predict exact capacity as it's based on many factors in Go runtime. 19 | testutil.Assert(t, n*7 <= cap(wp) && cap(wp) <= n*7+(8*1024)) 20 | 21 | p := createSlice_Better(n) 22 | testutil.Equals(t, wp, p) 23 | testutil.Equals(t, n*7, len(p)) 24 | testutil.Equals(t, len(p), cap(p)) 25 | } 26 | 27 | func BenchmarkCreateSlice(b *testing.B) { 28 | const n = int(1e4) 29 | 30 | b.Run("without prealloc", func(b *testing.B) { 31 | for i := 0; i < b.N; i++ { 32 | _ = createSlice(n) 33 | } 34 | }) 35 | b.Run("prealloc", func(b *testing.B) { 36 | for i := 0; i < b.N; i++ { 37 | _ = createSlice_Better(n) 38 | } 39 | }) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/emptystruct/dups.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package emptystruct 5 | 6 | // Example of simple optimization that allows to do less work which is not necessary. 7 | // Read more in "Efficient Go"; Example 11-1. 8 | 9 | func HasDuplicates[T comparable](slice ...T) bool { 10 | dup := make(map[T]any, len(slice)) 11 | for _, s := range slice { 12 | if _, ok := dup[s]; ok { 13 | return true 14 | } 15 | dup[s] = "whatever, I don't use this value" 16 | } 17 | return false 18 | } 19 | 20 | func HasDuplicates_Better[T comparable](slice ...T) bool { 21 | dup := make(map[T]struct{}, len(slice)) 22 | for _, s := range slice { 23 | if _, ok := dup[s]; ok { 24 | return true 25 | } 26 | dup[s] = struct{}{} 27 | } 28 | return false 29 | } 30 | 31 | func HasDuplicates_NonGeneric(slice ...float64) bool { 32 | dup := make(map[float64]struct{}, len(slice)) 33 | for _, s := range slice { 34 | if _, ok := dup[s]; ok { 35 | return true 36 | } 37 | dup[s] = struct{}{} 38 | } 39 | return false 40 | } 41 | -------------------------------------------------------------------------------- /pkg/getter/getter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package getter_test 5 | 6 | import ( 7 | "errors" 8 | "testing" 9 | 10 | "github.com/efficientgo/core/testutil" 11 | "github.com/efficientgo/examples/pkg/getter" 12 | ) 13 | 14 | type testReporter struct { 15 | r []getter.Report 16 | } 17 | 18 | func (r *testReporter) Get() []getter.Report { 19 | return r.r 20 | } 21 | 22 | type testReport struct { 23 | err error 24 | } 25 | 26 | func (r testReport) Error() error { 27 | return r.err 28 | } 29 | 30 | func TestFailureRatio(t *testing.T) { 31 | r := &testReporter{} 32 | 33 | ratio := getter.FailureRatio(r) 34 | testutil.Equals(t, 0., ratio) 35 | ratio = getter.FailureRatio_Better(r) 36 | testutil.Equals(t, 0., ratio) 37 | 38 | r.r = append( 39 | r.r, 40 | testReport{err: errors.New("a")}, 41 | testReport{err: errors.New("b")}, 42 | testReport{}, 43 | testReport{err: errors.New("d")}, 44 | ) 45 | ratio = getter.FailureRatio(r) 46 | testutil.Equals(t, 3/4., ratio) 47 | ratio = getter.FailureRatio_Better(r) 48 | testutil.Equals(t, 3/4., ratio) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/leak/http_exhaust.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package leak 5 | 6 | import ( 7 | "net/http" 8 | 9 | "github.com/efficientgo/core/errcapture" 10 | "github.com/efficientgo/core/errors" 11 | ) 12 | 13 | // Examples of code which is leaking resources, because of not exhausted readers. 14 | // Read more in "Efficient Go"; Example 11-10. 15 | 16 | func handleResp_Wrong(resp *http.Response) error { 17 | if resp.StatusCode != http.StatusOK { 18 | return errors.Newf("got non-200 response; code: %v", resp.StatusCode) 19 | } 20 | return nil 21 | } 22 | 23 | func handleResp_StillWrong(resp *http.Response) error { 24 | defer func() { 25 | _ = resp.Body.Close() 26 | }() 27 | if resp.StatusCode != http.StatusOK { 28 | return errors.Newf("got non-200 response; code: %v", resp.StatusCode) 29 | } 30 | return nil 31 | } 32 | 33 | func handleResp_Better(resp *http.Response) (err error) { 34 | defer errcapture.ExhaustClose(&err, resp.Body, "close") 35 | if resp.StatusCode != http.StatusOK { 36 | return errors.Newf("got non-200 response; code: %v", resp.StatusCode) 37 | } 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/memory/vars/vars_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package vars 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | "unsafe" 10 | ) 11 | 12 | func TestMyFunction(t *testing.T) { 13 | myFunction( 14 | 0, nil, 15 | biggie{}, nil, 16 | nil, nil, nil, 17 | nil, nil, 18 | ) 19 | 20 | b := make([][]byte, 10) 21 | fmt.Println(unsafe.Sizeof(b)) 22 | for i := range b { 23 | b[i] = make([]byte, 10) 24 | } 25 | 26 | fmt.Println(unsafe.Sizeof(b), unsafe.Sizeof(b[0])) 27 | 28 | fmt.Println(unsafe.Sizeof(string("123"))) 29 | } 30 | 31 | func test() { 32 | type A struct { //<1> 33 | Label1 int 34 | Label2 string 35 | Label3 []int 36 | } 37 | 38 | var label1 A // <2> 39 | label1.Label2 = "some string" // <3> 40 | 41 | var label2, label3, label4 *A // <2> 42 | label2 = &A{Label3: []int{1, 2}} // <4> 43 | label3 = label2 // <5> 44 | label4 = &label1 // <5> 45 | 46 | fmt.Println(label1, label2, label3, label4, unsafe.Sizeof(A{})) 47 | fmt.Printf("%T", label2) 48 | } 49 | 50 | func TestTest(t *testing.T) { 51 | test() 52 | } 53 | -------------------------------------------------------------------------------- /pkg/getter/getter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package getter 5 | 6 | type Report interface { 7 | Error() error 8 | } 9 | 10 | type ReportGetter interface { 11 | Get() []Report 12 | } 13 | 14 | // FailureRatio represents slightly less efficient (also less safe) use of getters. 15 | // Read more in "Efficient Go"; Example 1-1. 16 | func FailureRatio(reports ReportGetter) float64 { 17 | if len(reports.Get()) == 0 { 18 | return 0 19 | } 20 | 21 | var sum float64 22 | for _, report := range reports.Get() { 23 | if report.Error() != nil { 24 | sum++ 25 | } 26 | } 27 | return sum / float64(len(reports.Get())) 28 | } 29 | 30 | // FailureRatio_Better represents more efficient (also safer and more readable) use of getters. 31 | // Read more in "Efficient Go"; Example 1-2 (called `FailureRatio` in example). 32 | func FailureRatio_Better(reports ReportGetter) float64 { 33 | got := reports.Get() 34 | if len(got) == 0 { 35 | return 0 36 | } 37 | 38 | var sum float64 39 | for _, report := range got { 40 | if report.Error() != nil { 41 | sum++ 42 | } 43 | } 44 | return sum / float64(len(got)) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/godoc/block.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | // Copyright (c) The Thanos Authors. 5 | // Licensed under the Apache License 2.0. 6 | 7 | // Package block contains common functionality for interacting with TSDB blocks 8 | // in the context of Thanos. 9 | package block 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | 15 | "github.com/oklog/ulid" 16 | ) 17 | 18 | const ( 19 | // MetaFilename is the known JSON filename for meta information. 20 | MetaFilename = "meta.json" 21 | ) 22 | 23 | // Download downloads directory that is meant to be block directory. If any of the files 24 | // have a hash calculated in the meta file and it matches with what is in the destination path then 25 | // we do not download it. We always re-download the meta file. 26 | // BUG(bwplotka): No known bugs, but if there was one, it would be outlined here. 27 | func Download(ctx context.Context, id ulid.ULID, dst string) error { 28 | fmt.Println("downloaded") 29 | return nil 30 | } 31 | 32 | // cleanUp cleans the partially uploaded files. 33 | func cleanUp(ctx context.Context, id ulid.ULID) error { 34 | fmt.Println("cleaned") 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /pkg/generics/blocks.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package generics 5 | 6 | import ( 7 | "sort" 8 | "time" 9 | 10 | "github.com/google/uuid" 11 | ) 12 | 13 | type Block struct { 14 | id uuid.UUID 15 | start, end time.Time 16 | // ... 17 | } 18 | 19 | // Example of generic sorting function for any type that supports `Compare` method. 20 | // Read more in "Efficient Go"; Example 2-15. 21 | 22 | type Comparable[T any] interface { 23 | Compare(T) int 24 | } 25 | 26 | type genericSortable[T Comparable[T]] []T 27 | 28 | func (s genericSortable[T]) Len() int { return len(s) } 29 | func (s genericSortable[T]) Less(i, j int) bool { return s[i].Compare(s[j]) > 0 } 30 | func (s genericSortable[T]) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 31 | 32 | func genericSort[T Comparable[T]](slice []T) { 33 | sort.Sort(genericSortable[T](slice)) 34 | } 35 | 36 | // Compare is anti-pattern: For type parameters, prefer functions to methods. 37 | // https://go.dev/blog/when-generics 38 | func (b Block) Compare(other Block) int { 39 | if b.start.Before(other.start) { 40 | return 1 41 | } 42 | if b.start.Equal(other.start) { 43 | return 0 44 | } 45 | return -1 46 | } 47 | -------------------------------------------------------------------------------- /pkg/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import "github.com/efficientgo/core/errors" 7 | 8 | // Examples of error handling and different return arguments. 9 | // Read more in "Efficient Go"; Example 2-4. 10 | 11 | func shouldFail() bool { return false } 12 | 13 | func noErrCanHappen() int { 14 | // ... 15 | return 204 16 | } 17 | 18 | func doOrErr() error { 19 | // ... 20 | if shouldFail() { 21 | return errors.New("ups, XYZ failed") 22 | } 23 | return nil 24 | } 25 | 26 | func intOrErr() (int, error) { 27 | // ... 28 | if shouldFail() { 29 | return 0, errors.New("ups, XYZ2 failed") 30 | } 31 | return noErrCanHappen(), nil 32 | } 33 | 34 | // Examples of handling different return arguments. 35 | // Read more in "Efficient Go"; Example 2-5. 36 | 37 | func main() { 38 | ret := noErrCanHappen() 39 | if err := nestedDoOrErr(); err != nil { 40 | // handle error 41 | } 42 | ret2, err := intOrErr() 43 | if err != nil { 44 | // handle error 45 | } 46 | // ... 47 | 48 | _, _ = ret, ret2 // Just so we can compile the code. 49 | } 50 | 51 | func nestedDoOrErr() error { 52 | // ... 53 | if err := doOrErr(); err != nil { 54 | return errors.Wrap(err, "do") 55 | } 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /pkg/memory/mmap/mmap_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package mmap 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/efficientgo/core/testutil" 11 | ) 12 | 13 | func TestMemoryMappedFile(t *testing.T) { 14 | f, err := OpenFileBacked("./test_file.txt", 20) 15 | testutil.Ok(t, err) 16 | 17 | t.Cleanup(func() { 18 | testutil.Ok(t, f.Close()) 19 | }) 20 | 21 | testutil.Equals(t, "is is a test stri", string(f.Bytes()[2:19])) 22 | } 23 | 24 | func TestMemoryMappedFileAppend(t *testing.T) { 25 | f, err := OpenFileBacked("./test_file.txt", 20) 26 | testutil.Ok(t, err) 27 | 28 | t.Cleanup(func() { 29 | testutil.Ok(t, f.Close()) 30 | }) 31 | 32 | b := f.Bytes() 33 | 34 | // Writing to b causes signal SIGSEGV: segmentation violation 35 | // b[2] = 'd' 36 | 37 | fmt.Println(len(b), cap(b)) 38 | 39 | // Appending does not make much sense, since we replace memory mapped array by something on heap. But you can! 40 | b = append(b, '1') 41 | 42 | fmt.Println(len(b), cap(b)) 43 | } 44 | 45 | func TestMemoryMappedAnnonymous(t *testing.T) { 46 | f, err := OpenAnonymous(20) 47 | testutil.Ok(t, err) 48 | 49 | t.Cleanup(func() { 50 | testutil.Ok(t, f.Close()) 51 | }) 52 | 53 | b := f.Bytes() 54 | b[2] = 'd' 55 | 56 | fmt.Println(len(b), cap(b)) 57 | } 58 | -------------------------------------------------------------------------------- /pkg/compileroptimizeaway/opt_away_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package compileroptimizeaway 5 | 6 | import ( 7 | "math" 8 | "runtime" 9 | "testing" 10 | ) 11 | 12 | // BenchmarkPopcnt_Wrong is an example microbenchmark that can be optimized by compiler. 13 | // Read more in "Efficient Go"; Example 8-16. 14 | func BenchmarkPopcnt_Wrong(b *testing.B) { 15 | for i := 0; i < b.N; i++ { 16 | popcnt(math.MaxUint64) 17 | } 18 | } 19 | 20 | func BenchmarkPopcnt_Wrong2(b *testing.B) { 21 | for i := 0; i < b.N; i++ { 22 | popcnt(Input) 23 | } 24 | } 25 | 26 | var Sink uint64 27 | 28 | func BenchmarkPopcnt_Wrong3(b *testing.B) { 29 | var r uint64 30 | 31 | b.ResetTimer() 32 | for i := 0; i < b.N; i++ { 33 | r = popcnt(math.MaxUint64) 34 | } 35 | Sink = r 36 | } 37 | 38 | // BenchmarkPopcnt_Sink is one example on how we can countermeasure the problem visible in BenchmarkPopcnt_Wrong. 39 | // Read more in "Efficient Go"; Example 8-18. 40 | func BenchmarkPopcnt_Sink(b *testing.B) { 41 | var r uint64 42 | 43 | b.ResetTimer() 44 | for i := 0; i < b.N; i++ { 45 | r = popcnt(Input) 46 | } 47 | Sink = r 48 | } 49 | 50 | func BenchmarkPopcnt_KeepAlive(b *testing.B) { 51 | var r uint64 52 | 53 | for i := 0; i < b.N; i++ { 54 | r = popcnt(Input) 55 | } 56 | runtime.KeepAlive(r) 57 | } 58 | 59 | var Input uint64 = math.MaxUint64 60 | -------------------------------------------------------------------------------- /pkg/godoc/.bingo/Variables.mk: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.4.3. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST))) 4 | GOPATH ?= $(shell go env GOPATH) 5 | GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin 6 | GO ?= $(shell which go) 7 | 8 | # Below generated variables ensure that every time a tool under each variable is invoked, the correct version 9 | # will be used; reinstalling only if needed. 10 | # For example for bingo variable: 11 | # 12 | # In your main Makefile (for non array binaries): 13 | # 14 | #include .bingo/Variables.mk # Assuming -dir was set to .bingo . 15 | # 16 | #command: $(BINGO) 17 | # @echo "Running bingo" 18 | # @$(BINGO) 19 | # 20 | BINGO := $(GOBIN)/bingo-v0.4.3 21 | $(BINGO): $(BINGO_DIR)/bingo.mod 22 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 23 | @echo "(re)installing $(GOBIN)/bingo-v0.4.3" 24 | @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=bingo.mod -o=$(GOBIN)/bingo-v0.4.3 "github.com/bwplotka/bingo" 25 | 26 | GODOC := $(GOBIN)/godoc-v0.1.2 27 | $(GODOC): $(BINGO_DIR)/godoc.mod 28 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 29 | @echo "(re)installing $(GOBIN)/godoc-v0.1.2" 30 | @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=godoc.mod -o=$(GOBIN)/godoc-v0.1.2 "golang.org/x/tools/cmd/godoc" 31 | 32 | -------------------------------------------------------------------------------- /pkg/oop/oop.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package block 5 | 6 | import ( 7 | "fmt" 8 | "sort" 9 | "time" 10 | 11 | "github.com/google/uuid" 12 | ) 13 | 14 | // Production-like example of how Go can be used with object-oriented programming paradigm. 15 | // Read more in "Efficient Go"; Example 2-11. 16 | 17 | type Block struct { 18 | id uuid.UUID 19 | start, end time.Time 20 | // ... 21 | } 22 | 23 | func (b Block) Duration() time.Duration { 24 | return b.end.Sub(b.start) 25 | } 26 | 27 | func (b Block) String() string { 28 | return fmt.Sprint(b.id, ": ", b.start.Format(time.RFC3339), "-", b.end.Format(time.RFC3339)) 29 | } 30 | 31 | type Group struct { 32 | Block 33 | 34 | children []uuid.UUID 35 | } 36 | 37 | func (g *Group) Merge(b Block) { 38 | if g.end.IsZero() || g.end.Before(b.end) { 39 | g.end = b.end 40 | } 41 | if g.start.IsZero() || g.start.After(b.start) { 42 | g.start = b.start 43 | } 44 | 45 | g.children = append(g.children, b.id) 46 | 47 | // ... 48 | } 49 | 50 | func Compact(blocks ...Block) Block { 51 | sort.Sort(sortable(blocks)) 52 | 53 | g := &Group{} 54 | g.id = uuid.New() 55 | for _, b := range blocks { 56 | g.Merge(b) 57 | } 58 | return g.Block 59 | } 60 | 61 | type sortable []Block 62 | 63 | func (s sortable) Len() int { return len(s) } 64 | func (s sortable) Less(i, j int) bool { return s[i].start.Before(s[j].start) } 65 | func (s sortable) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 66 | 67 | var _ sort.Interface = sortable{} 68 | -------------------------------------------------------------------------------- /pkg/emptystruct/dups_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package emptystruct 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/efficientgo/core/testutil" 10 | ) 11 | 12 | func TestHasDuplicates(t *testing.T) { 13 | testutil.Equals(t, true, HasDuplicates[float64](1, 2, 3, 4, 5, 6, 1)) 14 | testutil.Equals(t, false, HasDuplicates[float64](1, 2, 3, 4, 5, 6, 7)) 15 | 16 | testutil.Equals(t, true, HasDuplicates_Better[float64](1, 2, 3, 4, 5, 6, 1)) 17 | testutil.Equals(t, false, HasDuplicates_Better[float64](1, 2, 3, 4, 5, 6, 7)) 18 | 19 | testutil.Equals(t, true, HasDuplicates_NonGeneric(1, 2, 3, 4, 5, 6, 1)) 20 | testutil.Equals(t, false, HasDuplicates_NonGeneric(1, 2, 3, 4, 5, 6, 7)) 21 | } 22 | 23 | func BenchmarkHasDuplicates(b *testing.B) { 24 | s := make([]float64, 1e6) 25 | for i := range s { 26 | s[i] = 99.0 + float64(i) 27 | } 28 | 29 | b.Run("HasDuplicates", func(b *testing.B) { 30 | b.ReportAllocs() 31 | 32 | b.ResetTimer() 33 | for i := 0; i < b.N; i++ { 34 | testutil.Equals(b, false, HasDuplicates[float64](s...)) 35 | } 36 | }) 37 | b.Run("HasDuplicates_Better", func(b *testing.B) { 38 | b.ReportAllocs() 39 | 40 | b.ResetTimer() 41 | for i := 0; i < b.N; i++ { 42 | testutil.Equals(b, false, HasDuplicates_Better[float64](s...)) 43 | } 44 | }) 45 | b.Run("HasDuplicates_NonGeneric", func(b *testing.B) { 46 | b.ReportAllocs() 47 | 48 | b.ResetTimer() 49 | for i := 0; i < b.N; i++ { 50 | testutil.Equals(b, false, HasDuplicates_NonGeneric(s...)) 51 | } 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/json/json.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package json 5 | 6 | import ( 7 | "encoding/json" 8 | "os" 9 | 10 | "github.com/efficientgo/core/errcapture" 11 | "github.com/efficientgo/core/errors" 12 | ) 13 | 14 | type Item struct { 15 | ID int `json:"id"` 16 | Name string `json:"name"` // Max 32 chars. 17 | Size [3]int `json:"size"` // Width, Height, Length. 18 | Weight int `json:"weight"` 19 | } 20 | 21 | // Read more in "Efficient Go"; Example 3-3 22 | 23 | type db struct { 24 | loaded []Item 25 | } 26 | 27 | func (d *db) load1(dbFile string) (err error) { 28 | f, err := os.Open(dbFile) 29 | if err != nil { 30 | return err 31 | } 32 | defer errcapture.Do(&err, f.Close, "close") 33 | 34 | return json.NewDecoder(f).Decode(&d.loaded) 35 | } 36 | 37 | func (d *db) load2(dbFile string) (err error) { 38 | f, err := os.Open(dbFile) 39 | if err != nil { 40 | return err 41 | } 42 | defer errcapture.Do(&err, f.Close, "close") 43 | 44 | st, err := f.Stat() 45 | if err != nil { 46 | return err 47 | } 48 | 49 | // Read at once, and unmarshal as a whole. 50 | b := make([]byte, int(st.Size())) 51 | n, err := f.Read(b) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | if n != int(st.Size()) { 57 | return errors.Wrapf(err, "read only %v/%v bytes", n, st.Size()) 58 | } 59 | 60 | // Estimate number of items based on assumption that we each item has roughly 61 | // 88 bytes. This is best effort--in a worse case we over allocate a little, 62 | // or we will have to resize once. 63 | d.loaded = make([]Item, 0, int((st.Size())-24)/88) 64 | return json.Unmarshal(b, &d.loaded) 65 | } 66 | -------------------------------------------------------------------------------- /pkg/profile/fd/fd.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package fd 5 | 6 | import ( 7 | "os" 8 | "runtime/pprof" 9 | ) 10 | 11 | // Example custom profile you can write on top `pprof.Profile` helper. 12 | // Read more in "Efficient Go"; Example 9-1. 13 | 14 | var fdProfile = pprof.NewProfile("fd.inuse") 15 | 16 | // File is a wrapper on os.File that tracks file descriptor lifetime. 17 | type File struct { 18 | *os.File 19 | } 20 | 21 | // Open opens file and tracks it in the `fd` profile`. 22 | // NOTE(bwplotka): We could use finalizers here, but explicit Close is more reliable and accurate. 23 | // Unfortunately it also changes type which might be dropped accidentally. 24 | func Open(name string) (*File, error) { 25 | f, err := os.Open(name) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | fdProfile.Add(f, 2) 31 | return &File{File: f}, nil 32 | } 33 | 34 | // Close closes files and updates profile. 35 | func (f *File) Close() error { 36 | defer fdProfile.Remove(f.File) 37 | return f.File.Close() 38 | } 39 | 40 | // Write saves the profile of the currently open file descriptors in to file in pprof format. 41 | func Write(profileOutPath string) error { 42 | out, err := os.Create(profileOutPath) // For simplicity, we don't include this file in profile. 43 | if err != nil { 44 | return err 45 | } 46 | if err := fdProfile.WriteTo(out, 0); err != nil { 47 | _ = out.Close() 48 | return err 49 | } 50 | return out.Close() 51 | } 52 | 53 | func CreateTemp(dir, pattern string) (*File, error) { 54 | f, err := os.CreateTemp(dir, pattern) 55 | if err != nil { 56 | return nil, err 57 | } 58 | fdProfile.Add(f, 2) 59 | return &File{File: f}, nil 60 | } 61 | -------------------------------------------------------------------------------- /.github/workflows/go.yaml: -------------------------------------------------------------------------------- 1 | name: go 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | pull_request: 9 | 10 | jobs: 11 | lint: 12 | runs-on: ubuntu-latest 13 | name: Linters (Static Analysis) for Go 14 | steps: 15 | - name: Checkout code into the Go module directory. 16 | uses: actions/checkout@v3 17 | 18 | - name: Install Go 19 | uses: actions/setup-go@v3 20 | with: 21 | go-version: 1.21.x 22 | 23 | - uses: actions/cache@v1 24 | with: 25 | path: ~/go/pkg/mod 26 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 27 | 28 | - name: Linting & vetting. 29 | env: 30 | GOBIN: /tmp/.bin 31 | run: make lint 32 | tests: 33 | runs-on: ${{ matrix.platform }} 34 | strategy: 35 | fail-fast: false 36 | matrix: 37 | go: ["1.20.x", "1.21.x"] 38 | platform: [ubuntu-latest, macos-latest] 39 | 40 | name: Unit tests on Go ${{ matrix.go }} ${{ matrix.platform }} 41 | steps: 42 | - name: Checkout code into the Go module directory. 43 | uses: actions/checkout@v3 44 | 45 | - name: Install Go 46 | uses: actions/setup-go@v3 47 | with: 48 | go-version: ${{ matrix.go }} 49 | 50 | - name: Install docker - MacOS 51 | if: runner.os == 'macOS' 52 | run: | 53 | brew install docker colima 54 | colima start 55 | 56 | - uses: actions/cache@v1 57 | with: 58 | path: ~/go/pkg/mod 59 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 60 | 61 | - name: Run unit tests. 62 | env: 63 | GOBIN: /tmp/.bin 64 | run: make test 65 | -------------------------------------------------------------------------------- /pkg/memory/mmap/mmap.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package mmap 5 | 6 | import ( 7 | "os" 8 | 9 | "github.com/efficientgo/core/errors" 10 | "github.com/efficientgo/core/merrors" 11 | "golang.org/x/sys/unix" 12 | ) 13 | 14 | // Wrapper for using memory mapping. 15 | // Read more in "Efficient Go"; Example 5-1. 16 | 17 | type MemoryMap struct { 18 | f *os.File // nil if anonymous. 19 | b []byte 20 | } 21 | 22 | func OpenFileBacked(path string, size int) (mf *MemoryMap, _ error) { 23 | f, err := os.Open(path) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | b, err := unix.Mmap(int(f.Fd()), 0, size, unix.PROT_READ, unix.MAP_SHARED) 29 | if err != nil { 30 | return nil, merrors.New(f.Close(), err).Err() 31 | } 32 | 33 | return &MemoryMap{f: f, b: b}, nil 34 | } 35 | 36 | func OpenAnonymous(size int) (mf *MemoryMap, _ error) { 37 | b, err := unix.Mmap(0, 0, size, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_PRIVATE|unix.MAP_ANON) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return &MemoryMap{f: nil, b: b}, nil 42 | } 43 | 44 | func (f *MemoryMap) Close() error { 45 | errs := merrors.New() 46 | errs.Add(unix.Munmap(f.b)) 47 | if f.f != nil { 48 | errs.Add(f.f.Close()) 49 | } 50 | return errs.Err() 51 | } 52 | 53 | func (f *MemoryMap) Bytes() []byte { return f.b } 54 | 55 | func (f *MemoryMap) File() *os.File { return f.f } 56 | 57 | func (f *MemoryMap) Advise(advise int) error { 58 | if f.f != nil { 59 | // TODO(bwplotka): Provide table what works in SHARED mode. 60 | return errors.New("Most of madvise calls works ony on MAP_ANON mappings.") 61 | } 62 | if err := unix.Madvise(f.b, advise); err != nil { 63 | return err 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/profile/fd/example/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "io" 8 | "log" 9 | "sync" 10 | 11 | "github.com/efficientgo/examples/pkg/profile/fd" 12 | ) 13 | 14 | // Example application instrumented with custom fd profile from Example 9-1. 15 | // Read more in "Efficient Go"; Example 9-2. 16 | 17 | type TestApp struct { 18 | files []io.ReadCloser 19 | } 20 | 21 | func (a *TestApp) Close() { 22 | for _, cl := range a.files { 23 | _ = cl.Close() // TODO: Check error. 24 | } 25 | a.files = a.files[:0] 26 | } 27 | 28 | func (a *TestApp) open(name string) { 29 | f, _ := fd.Open(name) // TODO: Check error. 30 | a.files = append(a.files, f) 31 | } 32 | 33 | func (a *TestApp) OpenSingleFile(name string) { 34 | a.open(name) 35 | } 36 | 37 | func (a *TestApp) OpenTenFiles(name string) { 38 | for i := 0; i < 10; i++ { 39 | a.open(name) 40 | } 41 | } 42 | 43 | func (a *TestApp) Open100FilesConcurrently(name string) { 44 | wg := sync.WaitGroup{} 45 | wg.Add(10) 46 | for i := 0; i < 10; i++ { 47 | go func() { 48 | a.OpenTenFiles(name) 49 | wg.Done() 50 | }() 51 | } 52 | wg.Wait() 53 | } 54 | 55 | func main() { 56 | a := &TestApp{} 57 | defer a.Close() 58 | 59 | // No matter how many files we opened in the past... 60 | for i := 0; i < 10; i++ { 61 | a.OpenTenFiles("/dev/null") 62 | a.Close() 63 | } 64 | 65 | // ...after last close, only files below will be used in profile. 66 | f, _ := fd.Open("/dev/null") // TODO: Check error. 67 | a.files = append(a.files, f) 68 | 69 | a.OpenSingleFile("/dev/null") 70 | a.OpenTenFiles("/dev/null") 71 | a.Open100FilesConcurrently("/dev/null") 72 | 73 | if err := fd.Write("fd.pprof"); err != nil { 74 | log.Fatal(err) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /pkg/sum/labeler/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | "time" 15 | 16 | "github.com/efficientgo/core/errors" 17 | "github.com/efficientgo/core/runutil" 18 | "github.com/efficientgo/core/testutil" 19 | "github.com/go-kit/log" 20 | ) 21 | 22 | func TestGetProfile(t *testing.T) { 23 | tmpDir := t.TempDir() 24 | errCh := make(chan error) 25 | 26 | ctx, cancel := context.WithCancel(context.Background()) 27 | t.Cleanup(func() { 28 | cancel() 29 | <-errCh 30 | }) 31 | 32 | go func() { 33 | errCh <- runMain(ctx, []string{`-objstore.config=type: FILESYSTEM 34 | config: 35 | directory: "."`}) 36 | close(errCh) 37 | }() 38 | 39 | rctx, rcancel := context.WithTimeout(context.Background(), 2*time.Minute) 40 | defer rcancel() 41 | testutil.Ok(t, runutil.RetryWithLog(log.NewLogfmtLogger(os.Stderr), 1*time.Second, rctx.Done(), func() error { 42 | res, err := http.Get("http://localhost:8080/debug/fgprof/profile?seconds=1") 43 | if err != nil { 44 | return err 45 | } 46 | 47 | f, err := os.Create(filepath.Join(tmpDir, "fgprof")) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | if _, err := io.Copy(f, res.Body); err != nil { 53 | _ = f.Close() 54 | return err 55 | } 56 | 57 | if err := f.Close(); err != nil { 58 | return err 59 | } 60 | 61 | fmt.Println(res.Status) 62 | 63 | if res.StatusCode != http.StatusOK { 64 | return errors.Newf("expected OK, got %v", res.StatusCode) 65 | } 66 | return nil 67 | })) 68 | 69 | select { 70 | case err := <-errCh: 71 | testutil.Ok(t, err) 72 | t.Fatal("expected to not fail") 73 | default: 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/prealloc/linkedlist.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package prealloc 5 | 6 | // Example of pre-allocation of linked list elements. 7 | // Read more in "Efficient Go"; Example 11-14. 8 | 9 | type Node struct { 10 | next *Node 11 | value int 12 | } 13 | 14 | type SinglyLinkedList struct { 15 | head *Node 16 | 17 | pool []Node 18 | poolIndex int 19 | } 20 | 21 | func (l *SinglyLinkedList) Grow(len int) { 22 | l.pool = make([]Node, len) 23 | l.poolIndex = 0 24 | } 25 | 26 | func (l *SinglyLinkedList) Insert(value int) { 27 | var newNode *Node 28 | if len(l.pool) > l.poolIndex { 29 | newNode = &l.pool[l.poolIndex] 30 | l.poolIndex++ 31 | } else { 32 | newNode = &Node{} 33 | } 34 | 35 | newNode.next = l.head 36 | newNode.value = value 37 | l.head = newNode 38 | } 39 | 40 | // Delete deletes node. However, this showcases kind-of leaking code. 41 | // Read more in "Efficient Go"; Example 11-15. 42 | func (l *SinglyLinkedList) Delete(n *Node) { 43 | if l.head == n { 44 | l.head = n.next 45 | return 46 | } 47 | 48 | for curr := l.head; curr != nil; curr = curr.next { 49 | if curr.next != n { 50 | continue 51 | } 52 | 53 | curr.next = n.next 54 | return 55 | } 56 | } 57 | 58 | // ClipMemory releases unused memory. 59 | // Read more in "Efficient Go"; Example 11-16. 60 | func (l *SinglyLinkedList) ClipMemory() { 61 | var objs int 62 | for curr := l.head; curr != nil; curr = curr.next { 63 | objs++ 64 | } 65 | 66 | l.pool = make([]Node, objs) 67 | l.poolIndex = 0 68 | for curr := l.head; curr != nil; curr = curr.next { 69 | oldCurr := curr 70 | curr = &l.pool[l.poolIndex] 71 | l.poolIndex++ 72 | 73 | curr.next = oldCurr.next 74 | curr.value = oldCurr.value 75 | 76 | if oldCurr == l.head { 77 | l.head = curr 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /.bingo/Variables.mk: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST))) 4 | GOPATH ?= $(shell go env GOPATH) 5 | GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin 6 | GO ?= $(shell which go) 7 | 8 | # Below generated variables ensure that every time a tool under each variable is invoked, the correct version 9 | # will be used; reinstalling only if needed. 10 | # For example for copyright variable: 11 | # 12 | # In your main Makefile (for non array binaries): 13 | # 14 | #include .bingo/Variables.mk # Assuming -dir was set to .bingo . 15 | # 16 | #command: $(COPYRIGHT) 17 | # @echo "Running copyright" 18 | # @$(COPYRIGHT) 19 | # 20 | COPYRIGHT := $(GOBIN)/copyright-v0.0.0-20210829154005-c7bad8450208 21 | $(COPYRIGHT): $(BINGO_DIR)/copyright.mod 22 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 23 | @echo "(re)installing $(GOBIN)/copyright-v0.0.0-20210829154005-c7bad8450208" 24 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=copyright.mod -o=$(GOBIN)/copyright-v0.0.0-20210829154005-c7bad8450208 "github.com/efficientgo/tools/copyright" 25 | 26 | GOLANGCI_LINT := $(GOBIN)/golangci-lint-v1.55.2 27 | $(GOLANGCI_LINT): $(BINGO_DIR)/golangci-lint.mod 28 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 29 | @echo "(re)installing $(GOBIN)/golangci-lint-v1.55.2" 30 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v1.55.2 "github.com/golangci/golangci-lint/cmd/golangci-lint" 31 | 32 | MDOX := $(GOBIN)/mdox-v0.9.0 33 | $(MDOX): $(BINGO_DIR)/mdox.mod 34 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 35 | @echo "(re)installing $(GOBIN)/mdox-v0.9.0" 36 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.9.0 "github.com/bwplotka/mdox" 37 | 38 | -------------------------------------------------------------------------------- /pkg/leak/file.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package leak 5 | 6 | import ( 7 | "io" 8 | "os" 9 | 10 | "github.com/efficientgo/core/errcapture" 11 | "github.com/efficientgo/core/logerrcapture" 12 | "github.com/efficientgo/core/merrors" 13 | "github.com/go-kit/log" 14 | "github.com/go-kit/log/level" 15 | ) 16 | 17 | // Example on common leaks using `os.File`. 18 | // Read more in "Efficient Go"; Example 11-8. 19 | 20 | func doWithFile_Wrong(fileName string) error { 21 | f, err := os.Open(fileName) 22 | if err != nil { 23 | return err 24 | } 25 | defer f.Close() // Wrong! 26 | 27 | // Use file... 28 | 29 | return nil 30 | } 31 | 32 | func doWithFile_LogCloseErr(logger log.Logger, fileName string) { 33 | f, err := os.Open(fileName) 34 | if err != nil { 35 | level.Error(logger).Log("err", err) 36 | return 37 | } 38 | defer logerrcapture.Do(logger, f.Close, "close file") 39 | 40 | // Use file... 41 | } 42 | 43 | func doWithFile_CaptureCloseErr(fileName string) (err error) { 44 | f, err := os.Open(fileName) 45 | if err != nil { 46 | return err 47 | } 48 | defer errcapture.Do(&err, f.Close, "close file") 49 | 50 | // Use file... 51 | 52 | return nil 53 | } 54 | 55 | // Example on common leaks using `os.File` when multiple files are used. 56 | // Read more in "Efficient Go"; Example 11-9. 57 | 58 | func openMultiple_Wrong(fileNames ...string) ([]io.ReadCloser, error) { 59 | files := make([]io.ReadCloser, 0, len(fileNames)) 60 | for _, fn := range fileNames { 61 | f, err := os.Open(fn) 62 | if err != nil { 63 | return nil, err // Leaked files! 64 | } 65 | files = append(files, f) 66 | } 67 | return files, nil 68 | } 69 | 70 | func openMultiple_Correct(fileNames ...string) ([]io.ReadCloser, error) { 71 | files := make([]io.ReadCloser, 0, len(fileNames)) 72 | for _, fn := range fileNames { 73 | f, err := os.Open(fn) 74 | if err != nil { 75 | return nil, merrors.New(err, closeAll(files)).Err() 76 | } 77 | files = append(files, f) 78 | } 79 | return files, nil 80 | } 81 | 82 | func closeAll(closers []io.ReadCloser) error { 83 | errs := merrors.New() 84 | for _, c := range closers { 85 | errs.Add(c.Close()) 86 | } 87 | return errs.Err() 88 | } 89 | -------------------------------------------------------------------------------- /pkg/pools/reuse_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package pools 5 | 6 | import ( 7 | "bytes" 8 | "io" 9 | "runtime" 10 | "sync" 11 | "testing" 12 | 13 | "github.com/efficientgo/core/testutil" 14 | ) 15 | 16 | func TestReuse(t *testing.T) { 17 | t.Run("", func(t *testing.T) { 18 | buf := make([]byte, 1e3) 19 | 20 | r := bytes.NewReader([]byte("abc")) 21 | w := &bytes.Buffer{} 22 | n, err := io.CopyBuffer(w, r, buf) 23 | 24 | testutil.Ok(t, err) 25 | testutil.Equals(t, 3, int(n)) 26 | testutil.Equals(t, "abc", w.String()) 27 | }) 28 | t.Run("", func(t *testing.T) { 29 | buf := make([]byte, 1e3) 30 | 31 | r := bytes.NewReader([]byte("abc")) 32 | n, err := io.ReadFull(r, buf[:3]) 33 | 34 | testutil.Ok(t, err) 35 | testutil.Equals(t, 3, int(n)) 36 | testutil.Equals(t, "abc", string(buf[:n])) 37 | }) 38 | } 39 | 40 | // BenchmarkProcess shows benchmarks of buffer that highlights common bug. 41 | // Read more in "Efficient Go"; Example 11-20. 42 | func BenchmarkProcess(b *testing.B) { 43 | b.Run("alloc", func(b *testing.B) { 44 | b.ReportAllocs() 45 | for i := 0; i < b.N; i++ { 46 | processUsingBuffer(nil) 47 | } 48 | }) 49 | b.Run("buffer", func(b *testing.B) { 50 | b.ReportAllocs() 51 | buf := make([]byte, 1e6) 52 | b.ResetTimer() 53 | for i := 0; i < b.N; i++ { 54 | processUsingBuffer(buf) 55 | } 56 | }) 57 | b.Run("pool-wrong", func(b *testing.B) { 58 | b.ReportAllocs() 59 | 60 | p := sync.Pool{ 61 | New: func() any { return []byte{} }, 62 | } 63 | b.ResetTimer() 64 | for i := 0; i < b.N; i++ { 65 | processUsingPool_Wrong(&p) 66 | } 67 | }) 68 | b.Run("pool", func(b *testing.B) { 69 | b.ReportAllocs() 70 | 71 | p := sync.Pool{ 72 | New: func() any { return []byte{} }, 73 | } 74 | b.ResetTimer() 75 | for i := 0; i < b.N; i++ { 76 | processUsingPool(&p) 77 | } 78 | }) 79 | b.Run("pool-GC", func(b *testing.B) { 80 | b.ReportAllocs() 81 | 82 | p := sync.Pool{ 83 | New: func() any { return []byte{} }, 84 | } 85 | b.ResetTimer() 86 | for i := 0; i < b.N; i++ { 87 | processUsingPool(&p) 88 | runtime.GC() 89 | runtime.GC() 90 | } 91 | }) 92 | b.Run("buffer-GC", func(b *testing.B) { 93 | b.ReportAllocs() 94 | buf := make([]byte, 1e6) 95 | b.ResetTimer() 96 | for i := 0; i < b.N; i++ { 97 | processUsingBuffer(buf) 98 | runtime.GC() 99 | runtime.GC() 100 | } 101 | }) 102 | } 103 | -------------------------------------------------------------------------------- /pkg/leak/http_close.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package leak 5 | 6 | import ( 7 | "context" 8 | "net/http" 9 | "strconv" 10 | "time" 11 | ) 12 | 13 | // Example case of leak in HTTP handlers. 14 | // Read more in "Efficient Go"; Example 11-2. 15 | 16 | func ComplexComputation() int { 17 | time.Sleep(1 * time.Second) // Computation. 18 | time.Sleep(1 * time.Second) // Cleanup. 19 | return 4 20 | } 21 | 22 | func Handle_VeryWrong(w http.ResponseWriter, r *http.Request) { 23 | respCh := make(chan int) 24 | 25 | go func() { 26 | defer close(respCh) 27 | respCh <- ComplexComputation() 28 | }() 29 | 30 | // Some other work... 31 | 32 | select { 33 | case <-r.Context().Done(): 34 | return 35 | case resp := <-respCh: 36 | _, _ = w.Write([]byte(strconv.Itoa(resp))) 37 | return 38 | } 39 | } 40 | 41 | // More examples of leaks in HTTP handlers. 42 | // Read more in "Efficient Go"; Example 11-5. 43 | 44 | func Handle_Wrong(w http.ResponseWriter, r *http.Request) { 45 | respCh := make(chan int, 1) 46 | 47 | go func() { 48 | defer close(respCh) 49 | respCh <- ComplexComputation() 50 | }() 51 | 52 | // Some other work... 53 | 54 | select { 55 | case <-r.Context().Done(): 56 | return 57 | case resp := <-respCh: 58 | _, _ = w.Write([]byte(strconv.Itoa(resp))) 59 | return 60 | } 61 | } 62 | 63 | func ComplexComputationWithCtx(ctx context.Context) (ret int) { 64 | select { 65 | case <-ctx.Done(): 66 | case <-time.After(1 * time.Second): // Computation. 67 | ret = 4 68 | } 69 | 70 | time.Sleep(1 * time.Second) // Cleanup. 71 | return ret 72 | } 73 | 74 | func Handle_AlsoWrong(w http.ResponseWriter, r *http.Request) { 75 | respCh := make(chan int, 1) 76 | 77 | go func() { 78 | defer close(respCh) 79 | respCh <- ComplexComputationWithCtx(r.Context()) 80 | }() 81 | 82 | // Some other work... 83 | 84 | select { 85 | case <-r.Context().Done(): 86 | return 87 | case resp := <-respCh: 88 | _, _ = w.Write([]byte(strconv.Itoa(resp))) 89 | return 90 | } 91 | } 92 | 93 | // Recommended code that does not leak. 94 | // Read more in "Efficient Go"; Example 11-6. 95 | 96 | func Handle_Better(w http.ResponseWriter, r *http.Request) { 97 | respCh := make(chan int) 98 | 99 | go func() { 100 | defer close(respCh) 101 | respCh <- ComplexComputationWithCtx(r.Context()) 102 | }() 103 | 104 | // Some other work... 105 | 106 | resp := <-respCh 107 | if r.Context().Err() != nil { 108 | return 109 | } 110 | 111 | _, _ = w.Write([]byte(strconv.Itoa(resp))) 112 | } 113 | -------------------------------------------------------------------------------- /pkg/concurrency/concurrency.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package concurrency 5 | 6 | import ( 7 | "math/rand" 8 | "sync" 9 | "sync/atomic" 10 | ) 11 | 12 | // Simplest example of goroutines. 13 | // Read more in "Efficient Go"; Example 4-5. 14 | 15 | func anotherFunction(arg1 string) { /*...*/ } 16 | 17 | func function() { 18 | // Scope of the current goroutine. 19 | // ... 20 | 21 | go func() { 22 | // This scope will run concurrently any moment now. 23 | // ... 24 | }() 25 | 26 | // anotherFunction will run concurrently any moment now. 27 | go anotherFunction("argument1") 28 | 29 | // After our function ends, two goroutines we started can still run. 30 | return 31 | } 32 | 33 | var randInt64 = func() int64 { 34 | return rand.Int63() 35 | } 36 | 37 | // Example of communicating state between goroutines using atomic operations. 38 | // Read more in "Efficient Go"; Example 4-6. 39 | func sharingWithAtomic() (sum int64) { 40 | var wg sync.WaitGroup 41 | 42 | concurrentFn := func() { 43 | // ... 44 | atomic.AddInt64(&sum, randInt64()) 45 | wg.Done() 46 | } 47 | wg.Add(3) 48 | go concurrentFn() 49 | go concurrentFn() 50 | go concurrentFn() 51 | 52 | wg.Wait() 53 | return sum 54 | } 55 | 56 | // Example of communicating state between goroutines using mutex locking. 57 | // Read more in "Efficient Go"; Example 4-7. 58 | func sharingWithMutex() (sum int64) { 59 | var wg sync.WaitGroup 60 | var mu sync.Mutex 61 | 62 | concurrentFn := func() { 63 | // ... 64 | mu.Lock() 65 | sum += randInt64() 66 | mu.Unlock() 67 | wg.Done() 68 | } 69 | wg.Add(3) 70 | go concurrentFn() 71 | go concurrentFn() 72 | go concurrentFn() 73 | 74 | wg.Wait() 75 | return sum 76 | } 77 | 78 | // Example of communicating state between goroutines using mutex locking. 79 | // Read more in "Efficient Go"; Example 4-8. 80 | func sharingWithChannel() (sum int64) { 81 | result := make(chan int64) 82 | 83 | concurrentFn := func() { 84 | // ... 85 | result <- randInt64() 86 | } 87 | go concurrentFn() 88 | go concurrentFn() 89 | go concurrentFn() 90 | 91 | for i := 0; i < 3; i++ { 92 | sum += <-result 93 | } 94 | close(result) 95 | return sum 96 | } 97 | 98 | // Example of communicating state between goroutines using sharded space. 99 | func sharingWithShardedSpace() (sum int64) { 100 | var wg sync.WaitGroup 101 | results := [3]int64{} 102 | 103 | concurrentFn := func(i int) { 104 | // ... 105 | results[i] = randInt64() 106 | wg.Done() 107 | } 108 | wg.Add(3) 109 | go concurrentFn(0) 110 | go concurrentFn(1) 111 | go concurrentFn(2) 112 | 113 | wg.Wait() 114 | for _, res := range results { 115 | sum += res 116 | } 117 | return sum 118 | } 119 | -------------------------------------------------------------------------------- /pkg/metrics/dummy.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package metrics 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | "math/rand" 11 | "net/http" 12 | "net/http/httptest" 13 | "runtime" 14 | "time" 15 | 16 | "github.com/bwplotka/tracing-go/tracing" 17 | "github.com/prometheus/client_golang/prometheus" 18 | "github.com/prometheus/client_golang/prometheus/promhttp" 19 | ) 20 | 21 | func prepare() { fmt.Println("initializing operation!") } 22 | 23 | //nolint 24 | func doOperation() error { 25 | // Do some dummy, randomized heavy work (both in terms of latency, CPU and memory usage). 26 | alloc := make([]byte, 1e6) 27 | for i := 0; i < int(rand.Float64()*100); i++ { 28 | _ = fmt.Sprintf("doing stuff! %+v", alloc) 29 | } 30 | 31 | runtime.GC() // To have more interesting GC metrics. 32 | 33 | switch rand.Intn(3) { 34 | case 0: 35 | return nil 36 | case 1: 37 | return errors.New("error first") 38 | case 2: 39 | return errors.New("error other") 40 | } 41 | return nil 42 | } 43 | 44 | func doOperationWithCtx(ctx context.Context) error { 45 | _, span := tracing.StartSpan(ctx, "first operation") 46 | // Do some dummy, randomized heavy work (both in terms of latency, CPU and memory usage). 47 | alloc := make([]byte, 1e6) 48 | for i := 0; i < int(rand.Float64()*125); i++ { 49 | _ = fmt.Sprintf("doing stuff! %+v", alloc) // Hope for this to not get cleared by compiler. 50 | } 51 | span.End(nil) 52 | 53 | runtime.GC() // To have more interesting GC metrics. 54 | 55 | _ = tracing.DoInSpan(ctx, "sub operation2", func(ctx context.Context) error { 56 | return nil 57 | }) 58 | 59 | _ = tracing.DoInSpan(ctx, "sub operation3", func(ctx context.Context) error { 60 | return nil 61 | }) 62 | 63 | return tracing.DoInSpan( 64 | ctx, 65 | "choosing error", 66 | func(ctx context.Context) error { 67 | switch rand.Intn(3) { 68 | default: 69 | return nil 70 | case 1: 71 | time.Sleep(300 * time.Millisecond) // For more interesting results. 72 | return errors.New("error first") 73 | case 2: 74 | return errors.New("error other") 75 | } 76 | }) 77 | } 78 | 79 | func tearDown() { fmt.Println("closing operation!") } 80 | 81 | func errorType(err error) string { 82 | if err != nil { 83 | if err.Error() == "error first" { 84 | return "error1" 85 | } 86 | return "other_error" 87 | } 88 | return "" 89 | } 90 | 91 | func printPrometheusMetrics(reg prometheus.Gatherer) { 92 | rec := httptest.NewRecorder() 93 | promhttp.HandlerFor(reg, promhttp.HandlerOpts{DisableCompression: true, EnableOpenMetrics: true}).ServeHTTP(rec, &http.Request{}) 94 | if rec.Code != 200 { 95 | panic("unexpected error") 96 | } 97 | 98 | fmt.Println(rec.Body.String()) 99 | } 100 | -------------------------------------------------------------------------------- /pkg/memory/mmap/interactive/interactive_mmap.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "os" 10 | 11 | "github.com/efficientgo/examples/pkg/memory/mmap" 12 | ) 13 | 14 | // Creating memory mapping for 600MB of file. 15 | // Read more in "Efficient Go"; Example 5-3. 16 | 17 | func runMmap() { 18 | fmt.Println("PID", os.Getpid()) 19 | 20 | // TODO(bwplotka): Create big file here, so we can play with it - there is no need to upload so big file to GitHub. 21 | 22 | // Mmap 600 MB of 686MB file. 23 | f, err := mmap.OpenFileBacked("test686mbfile.out", 600*1024*1024) 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | 28 | // Check out: 29 | // ls -l /proc//map_files 30 | // watch -n 1 'export PID=$(ps -ax --format=pid,command | grep "exe/interactive" | grep -v "grep" | head -n 1 | cut -d" " -f2) && ps -ax --format=pid,rss,vsz | grep $PID && cat /proc/$PID/smaps | grep -A22 test686mbfile | grep Rss' 31 | b := f.Bytes() 32 | 33 | fmt.Println("1") 34 | fmt.Scanln() // wait for Enter Key 35 | 36 | fmt.Println("Reading 5000 index", b[5000]) 37 | 38 | fmt.Println("2") 39 | fmt.Scanln() // wait for Enter Key 40 | 41 | fmt.Println("Reading 100 000 index", b[100000]) 42 | 43 | fmt.Println("3") 44 | fmt.Scanln() // wait for Enter Key 45 | 46 | fmt.Println("Reading 104 000 index", b[104000]) 47 | 48 | fmt.Println("4") 49 | fmt.Scanln() // wait for Enter Key 50 | 51 | fmt.Println("Unmapping") 52 | if err := f.Close(); err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | fmt.Println("Finish") 57 | fmt.Scanln() // wait for Enter Key 58 | 59 | } 60 | 61 | func bookExample1() error { 62 | // Mmap 600MB of 686MB file. 63 | f, err := mmap.OpenFileBacked("test686mbfile.out", 600*1024*1024) 64 | if err != nil { 65 | return err 66 | } 67 | b := f.Bytes() 68 | 69 | // At this point we can see symlink to test686mbfile.out file in /proc//map_files. 70 | // If we would pause the program now `cat /proc//smaps | grep -A22 test686mbfile | grep Rss` shows 0KB. 71 | // NOTE exact RSS can vary due to state of memory, OS, OS version etc. 72 | fmt.Println("Reading 5000 index", b[5000]) 73 | 74 | // If we would pause the program now `cat /proc//smaps | grep -A22 test686mbfile | grep Rss` shows 48-70KB. 75 | 76 | fmt.Println("Reading 100 000 index", b[100000]) 77 | 78 | // If we would pause the program now `cat /proc//smaps | grep -A22 test686mbfile | grep Rss` shows 100-126KB. 79 | 80 | fmt.Println("Reading 104 000 index", b[104000]) 81 | 82 | // If we would pause the program now `cat /proc//smaps | grep -A22 test686mbfile | grep Rss` shows 100-126KB (same). 83 | 84 | if err := f.Close(); err != nil { 85 | return err 86 | } 87 | 88 | // If we would pause the program now `cat /proc//smaps | grep -A22 test686mbfile | grep Rss` shows nothing, RSS freed. 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /pkg/godoc/.bingo/godoc.sum: -------------------------------------------------------------------------------- 1 | github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= 2 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 3 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 4 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 5 | golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= 6 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 7 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 8 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 9 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= 10 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 11 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 12 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 13 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 14 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 15 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 16 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 17 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= 18 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 19 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 20 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 21 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 22 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 23 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 24 | golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= 25 | golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 26 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 27 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 28 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 29 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 30 | -------------------------------------------------------------------------------- /pkg/json/json_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package json 5 | 6 | import ( 7 | "bytes" 8 | "encoding/json" 9 | "fmt" 10 | "os" 11 | "testing" 12 | 13 | "github.com/efficientgo/core/testutil" 14 | ) 15 | 16 | func generateTestData(t testing.TB, count int) []Item { 17 | t.Helper() 18 | 19 | items := make([]Item, count) 20 | for i := 0; i < len(items); i += 5 { 21 | c := i 22 | items[c] = Item{ 23 | ID: c, 24 | Name: fmt.Sprintf("T-shirt ABC (%d)", i), // 18.8 25 | Size: [3]int{54, 49, 9}, 26 | Weight: 500, 27 | } 28 | c++ 29 | items[c] = Item{ 30 | ID: c, 31 | Name: fmt.Sprintf("Hoodie (%d)", i), 32 | Size: [3]int{64, 55, 15}, 33 | Weight: 700, 34 | } 35 | c++ 36 | items[c] = Item{ 37 | ID: c, 38 | Name: fmt.Sprintf("Mug (%d)", i), 39 | Size: [3]int{10, 8, 8}, 40 | Weight: 300, 41 | } 42 | c++ 43 | items[c] = Item{ 44 | ID: c, 45 | Name: fmt.Sprintf("Water Bottle (%d)", i), 46 | Size: [3]int{25, 7, 7}, 47 | Weight: 200, 48 | } 49 | c++ 50 | items[c] = Item{ 51 | ID: c, 52 | Name: fmt.Sprintf("Phone Case (%d)", i), 53 | Size: [3]int{15, 8, 3}, 54 | Weight: 100, 55 | } 56 | } 57 | return items 58 | } 59 | 60 | const testFile = "./test_db.1e6.json" 61 | 62 | func TestGenerateTestFile(t *testing.T) { 63 | t.Skip("only to generate a test file") 64 | items := generateTestData(t, 1e6) 65 | 66 | o, err := os.Create(testFile) 67 | testutil.Ok(t, err) 68 | defer func() { _ = o.Close() }() 69 | testutil.Ok(t, json.NewEncoder(o).Encode(items)) 70 | } 71 | 72 | func TestLoad(t *testing.T) { 73 | d := &db{} 74 | for _, tc := range []func(string) error{ 75 | d.load1, 76 | d.load2, 77 | } { 78 | t.Run("", func(t *testing.T) { 79 | inputFile := t.TempDir() + "/db_10k.json" 80 | 81 | items := generateTestData(t, 10e3) 82 | toCheck := items[39] 83 | 84 | o, err := os.Create(inputFile) 85 | testutil.Ok(t, err) 86 | defer func() { _ = o.Close() }() 87 | testutil.Ok(t, json.NewEncoder(o).Encode(items)) 88 | 89 | d.loaded = nil 90 | testutil.Ok(t, tc(inputFile)) 91 | testutil.Equals(t, toCheck, d.loaded[39]) 92 | }) 93 | } 94 | } 95 | 96 | func BenchmarkLoad(b *testing.B) { 97 | d := &db{} 98 | b.ReportAllocs() 99 | b.ResetTimer() 100 | for i := 0; i < b.N; i++ { 101 | d.loaded = nil 102 | testutil.Ok(b, d.load1(testFile)) 103 | } 104 | } 105 | 106 | func BenchmarkJSONUnmarshal(b *testing.B) { 107 | // 10e6 = 9s -> Xeon 21s/35s 108 | // 10e3 = 8.5ms -> Xeon 21.5ms 109 | items := generateTestData(b, 10e3) 110 | buf := &bytes.Buffer{} 111 | testutil.Ok(b, json.NewEncoder(buf).Encode(items)) 112 | 113 | b.ReportAllocs() 114 | b.ResetTimer() 115 | for i := 0; i < b.N; i++ { 116 | items = nil 117 | testutil.Ok(b, json.Unmarshal(buf.Bytes(), &items)) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /pkg/leak/http_close_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package leak 5 | 6 | import ( 7 | "context" 8 | "net/http" 9 | "net/http/httptest" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/efficientgo/core/testutil" 15 | "go.uber.org/goleak" 16 | ) 17 | 18 | // TestHandleCancel tests against leaks when cancelling the request. 19 | // Read more in "Efficient Go"; Example 11-3. 20 | func TestHandleCancel(t *testing.T) { 21 | defer goleak.VerifyNone(t) 22 | 23 | w := httptest.NewRecorder() 24 | r := httptest.NewRequest("", "https://efficientgo.com", nil) 25 | 26 | wg := sync.WaitGroup{} 27 | wg.Add(1) 28 | 29 | ctx, cancel := context.WithCancel(context.Background()) 30 | go func() { 31 | Handle_Better(w, r.WithContext(ctx)) 32 | wg.Done() 33 | }() 34 | // Immediately cancel. 35 | cancel() 36 | 37 | time.Sleep(3 * time.Second) 38 | wg.Wait() 39 | } 40 | 41 | func TestHandle(t *testing.T) { 42 | defer goleak.VerifyNone(t) 43 | 44 | t.Run("Handle_VeryWrong", func(t *testing.T) { 45 | w := httptest.NewRecorder() 46 | r := httptest.NewRequest("", "https://efficientgo.com", nil) 47 | Handle_VeryWrong(w, r) 48 | 49 | testutil.Equals(t, http.StatusOK, w.Code) 50 | testutil.Equals(t, "4", w.Body.String()) 51 | }) 52 | 53 | t.Run("Handle_Wrong", func(t *testing.T) { 54 | w := httptest.NewRecorder() 55 | r := httptest.NewRequest("", "https://efficientgo.com", nil) 56 | Handle_Wrong(w, r) 57 | 58 | testutil.Equals(t, http.StatusOK, w.Code) 59 | testutil.Equals(t, "4", w.Body.String()) 60 | }) 61 | 62 | t.Run("Handle_AlsoWrong", func(t *testing.T) { 63 | w := httptest.NewRecorder() 64 | r := httptest.NewRequest("", "https://efficientgo.com", nil) 65 | Handle_AlsoWrong(w, r) 66 | 67 | testutil.Equals(t, http.StatusOK, w.Code) 68 | testutil.Equals(t, "4", w.Body.String()) 69 | }) 70 | 71 | t.Run("Handle_Better", func(t *testing.T) { 72 | w := httptest.NewRecorder() 73 | r := httptest.NewRequest("", "https://efficientgo.com", nil) 74 | Handle_Better(w, r) 75 | 76 | testutil.Equals(t, http.StatusOK, w.Code) 77 | testutil.Equals(t, "4", w.Body.String()) 78 | }) 79 | } 80 | 81 | // Example of leaks in benchmarks (also wrong results). 82 | // Read more in "Efficient Go"; Example 11-7. 83 | 84 | func BenchmarkComplexComputation_Wrong(b *testing.B) { 85 | for i := 0; i < b.N; i++ { 86 | go func() { ComplexComputation() }() 87 | go func() { ComplexComputation() }() 88 | } 89 | } 90 | 91 | func BenchmarkComplexComputation_Better(b *testing.B) { 92 | defer goleak.VerifyNone( 93 | b, 94 | goleak.IgnoreTopFunction("testing.(*B).run1"), 95 | goleak.IgnoreTopFunction("testing.(*B).doBench"), 96 | ) 97 | 98 | b.ResetTimer() 99 | for i := 0; i < b.N; i++ { 100 | wg := sync.WaitGroup{} 101 | wg.Add(2) 102 | 103 | go func() { 104 | defer wg.Done() 105 | ComplexComputation() 106 | }() 107 | go func() { 108 | defer wg.Done() 109 | ComplexComputation() 110 | }() 111 | 112 | wg.Wait() 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | linters: 2 | # Disable specific linter 3 | # https://golangci-lint.run/usage/linters/#disabled-by-default 4 | disable: 5 | - asasalint 6 | - asciicheck 7 | - bidichk 8 | - bodyclose 9 | - containedctx 10 | - contextcheck 11 | - cyclop 12 | - deadcode 13 | - decorder 14 | - depguard 15 | - dogsled 16 | - dupl 17 | - dupword 18 | - durationcheck 19 | - errcheck 20 | - errchkjson 21 | - errname 22 | - errorlint 23 | - execinquery 24 | - exhaustive 25 | - exhaustivestruct 26 | - exhaustruct 27 | - exportloopref 28 | - forbidigo 29 | - forcetypeassert 30 | - funlen 31 | - gci 32 | - ginkgolinter 33 | - gocheckcompilerdirectives 34 | - gochecknoglobals 35 | - gochecknoinits 36 | - gochecksumtype 37 | - gocognit 38 | - goconst 39 | - gocritic 40 | - gocyclo 41 | - godot 42 | - godox 43 | - goerr113 44 | - gofmt 45 | - gofumpt 46 | - goheader 47 | - goimports 48 | - golint 49 | - gomnd 50 | - gomoddirectives 51 | - gomodguard 52 | - goprintffuncname 53 | - gosec 54 | - gosimple 55 | - gosmopolitan 56 | - govet 57 | - grouper 58 | - ifshort 59 | - importas 60 | - inamedparam 61 | - ineffassign 62 | - interfacebloat 63 | - interfacer 64 | - ireturn 65 | - lll 66 | - loggercheck 67 | - maintidx 68 | - makezero 69 | - maligned 70 | - mirror 71 | - misspell 72 | - musttag 73 | - nakedret 74 | - nestif 75 | - nilerr 76 | - nilnil 77 | - nlreturn 78 | - noctx 79 | - nolintlint 80 | - nonamedreturns 81 | - nosnakecase 82 | - nosprintfhostport 83 | - paralleltest 84 | - perfsprint 85 | - prealloc 86 | - predeclared 87 | - promlinter 88 | - protogetter 89 | - reassign 90 | - revive 91 | - rowserrcheck 92 | - scopelint 93 | - sloglint 94 | - sqlclosecheck 95 | - staticcheck 96 | - structcheck 97 | - stylecheck 98 | - tagalign 99 | - tagliatelle 100 | - tenv 101 | - testableexamples 102 | - testifylint 103 | - testpackage 104 | - thelper 105 | - tparallel 106 | - typecheck 107 | - unconvert 108 | - unparam 109 | - unused 110 | - usestdlibvars 111 | - varcheck 112 | - varnamelen 113 | - wastedassign 114 | - whitespace 115 | - wrapcheck 116 | - wsl 117 | - zerologlint 118 | # Run only fast linters from enabled linters set (first run won't be fast) 119 | # Default: false 120 | fast: true 121 | linters-settings: 122 | govet: 123 | # Settings per analyzer. 124 | settings: 125 | # Analyzer name, run `go tool vet help` to see all analyzers. 126 | # (in addition to default: 127 | # appends, asmdecl, assign, atomic, bools, buildtag, cgocall, composites, copylocks, defers, directive, errorsas, 128 | # framepointer, httpresponse, ifaceassert, loopclosure, lostcancel, nilfunc, printf, shift, sigchanyzer, slog, 129 | # stdmethods, stringintconv, structtag, testinggoroutine, tests, timeformat, unmarshal, unreachable, unsafeptr, 130 | # unusedresult 131 | # ). 132 | # (in addition to default 133 | # atomicalign, deepequalerrors, fieldalignment, findcall, nilness, reflectvaluecompare, shadow, sortslice, 134 | # timeformat, unusedwrite 135 | # ). 136 | # Run `go tool vet help` to see all analyzers. 137 | # Default: [] 138 | disable: 139 | - example 140 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include .bingo/Variables.mk 2 | 3 | MODULES ?= $(shell find $(PWD) -name "go.mod" | grep -v ".bingo" | xargs dirname) 4 | 5 | GO111MODULE ?= on 6 | export GO111MODULE 7 | 8 | GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin 9 | 10 | # Tools. 11 | GIT ?= $(shell which git) 12 | 13 | # Support gsed on OSX (installed via brew), falling back to sed. On Linux 14 | # systems gsed won't be installed, so will use sed as expected. 15 | SED ?= $(shell which gsed 2>/dev/null || which sed) 16 | 17 | define require_clean_work_tree 18 | @git update-index -q --ignore-submodules --refresh 19 | 20 | @if ! git diff-files --quiet --ignore-submodules --; then \ 21 | echo >&2 "$1: you have unstaged changes."; \ 22 | git diff-files --name-status -r --ignore-submodules -- >&2; \ 23 | echo >&2 "Please commit or stash them."; \ 24 | exit 1; \ 25 | fi 26 | 27 | @if ! git diff-index --cached --quiet HEAD --ignore-submodules --; then \ 28 | echo >&2 "$1: your index contains uncommitted changes."; \ 29 | git diff-index --cached --name-status -r --ignore-submodules HEAD -- >&2; \ 30 | echo >&2 "Please commit or stash them."; \ 31 | exit 1; \ 32 | fi 33 | 34 | endef 35 | 36 | help: ## Displays help. 37 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) 38 | 39 | .PHONY: all 40 | all: format 41 | 42 | .PHONY: deps 43 | deps: ## Cleans up deps for all modules 44 | @echo ">> running deps tidy for all modules: $(MODULES)" 45 | for dir in $(MODULES) ; do \ 46 | cd $${dir} && go mod tidy; \ 47 | done 48 | 49 | .PHONY: docs 50 | docs: $(MDOX) ## Generates config snippets and doc formatting. 51 | @echo ">> generating docs $(PATH)" 52 | $(MDOX) fmt *.md 53 | 54 | .PHONY: docker 55 | docker: 56 | @echo ">> building labeler docker file" 57 | @cd ./pkg/sum/labeler && CGO_ENABLED=0 GOOS=linux go build -o labeler . 58 | @cd ./pkg/sum/labeler && docker build -t labeler:test . 59 | 60 | .PHONY: format 61 | format: ## Formats Go code. 62 | format: $(GOIMPORTS) 63 | @echo ">> formatting all modules Go code: $(MODULES)" 64 | @$(GOIMPORTS) -w $(MODULES) 65 | 66 | .PHONY: test 67 | test: ## Runs all Go unit tests. 68 | @echo ">> running tests for all modules: $(MODULES)" 69 | for dir in $(MODULES) ; do \ 70 | cd $${dir} && go test -v -race ./...; \ 71 | done 72 | 73 | .PHONY: check-git 74 | check-git: 75 | ifneq ($(GIT),) 76 | @test -x $(GIT) || (echo >&2 "No git executable binary found at $(GIT)."; exit 1) 77 | else 78 | @echo >&2 "No git binary found."; exit 1 79 | endif 80 | 81 | # PROTIP: 82 | # Add 83 | # --cpu-profile-path string Path to CPU profile output file 84 | # --mem-profile-path string Path to memory profile output file 85 | # to debug big allocations during linting. 86 | lint: ## Runs various static analysis against our code. 87 | lint: $(GOLANGCI_LINT) $(COPYRIGHT) format docs check-git deps 88 | $(call require_clean_work_tree,"detected not clean master before running lint - run make lint and commit changes.") 89 | @echo ">> linting all of the Go files GOGC=${GOGC}" 90 | for dir in $(MODULES) ; do \ 91 | cd $${dir} && $(GOLANGCI_LINT) run; \ 92 | done 93 | @echo ">> ensuring Copyright headers" 94 | @$(COPYRIGHT) $(shell find . -name "*.go") 95 | $(call require_clean_work_tree,"detected files without copyright - run make lint and commit changes.") 96 | -------------------------------------------------------------------------------- /pkg/metrics/httpmidleware/middleware.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package httpmidleware 5 | 6 | import ( 7 | "net/http" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promauto" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | ) 13 | 14 | // Middleware auto instruments net/http HTTP. 15 | type Middleware interface { 16 | // WrapHandler wraps the given HTTP handler for instrumentation. 17 | WrapHandler(handlerName string, handler http.Handler) http.HandlerFunc 18 | } 19 | 20 | type nopMiddleware struct{} 21 | 22 | func (ins nopMiddleware) WrapHandler(_ string, handler http.Handler) http.HandlerFunc { 23 | return func(w http.ResponseWriter, r *http.Request) { 24 | handler.ServeHTTP(w, r) 25 | } 26 | } 27 | 28 | // NewNopMiddleware provides a Middleware which does nothing. 29 | func NewNopMiddleware() Middleware { 30 | return nopMiddleware{} 31 | } 32 | 33 | type middleware struct { 34 | reg prometheus.Registerer 35 | 36 | buckets []float64 37 | } 38 | 39 | // NewMiddleware provides HTTP metric Middleware. 40 | // Passing nil as buckets uses the default buckets. 41 | func NewMiddleware(reg prometheus.Registerer, buckets []float64) Middleware { 42 | if buckets == nil { 43 | buckets = []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720} 44 | } 45 | 46 | return &middleware{ 47 | reg: reg, 48 | buckets: buckets, 49 | } 50 | } 51 | 52 | // WrapHandler wraps the given HTTP handler for instrumentation: 53 | // * It registers four metric collectors (if not already done) and reports HTTP 54 | // metrics to the (newly or already) registered collectors: http_requests_total 55 | // (CounterVec), http_request_duration_seconds (Histogram), 56 | // http_request_size_bytes (Summary), http_response_size_bytes (Summary). Each 57 | // has a constant label named "handler" with the provided handlerName as 58 | // value. http_requests_total is a metric vector partitioned by HTTP method 59 | // (label name "method") and HTTP status code (label name "code"). 60 | func (ins *middleware) WrapHandler(handlerName string, handler http.Handler) http.HandlerFunc { 61 | reg := prometheus.WrapRegistererWith(prometheus.Labels{"handler": handlerName}, ins.reg) 62 | 63 | requestDuration := promauto.With(reg).NewHistogramVec( 64 | prometheus.HistogramOpts{ 65 | Name: "http_request_duration_seconds", 66 | Help: "Tracks the latencies for HTTP requests.", 67 | Buckets: ins.buckets, 68 | }, 69 | []string{"method", "code"}, 70 | ) 71 | requestSize := promauto.With(reg).NewSummaryVec( 72 | prometheus.SummaryOpts{ 73 | Name: "http_request_size_bytes", 74 | Help: "Tracks the size of HTTP requests.", 75 | }, 76 | []string{"method", "code"}, 77 | ) 78 | requestsTotal := promauto.With(reg).NewCounterVec( 79 | prometheus.CounterOpts{ 80 | Name: "http_requests_total", 81 | Help: "Tracks the number of HTTP requests.", 82 | }, []string{"method", "code"}, 83 | ) 84 | responseSize := promauto.With(reg).NewSummaryVec( 85 | prometheus.SummaryOpts{ 86 | Name: "http_response_size_bytes", 87 | Help: "Tracks the size of HTTP responses.", 88 | }, 89 | []string{"method", "code"}, 90 | ) 91 | 92 | base := promhttp.InstrumentHandlerRequestSize( 93 | requestSize, 94 | promhttp.InstrumentHandlerCounter( 95 | requestsTotal, 96 | promhttp.InstrumentHandlerResponseSize( 97 | responseSize, 98 | promhttp.InstrumentHandlerDuration( 99 | requestDuration, 100 | http.HandlerFunc(func(writer http.ResponseWriter, r *http.Request) { 101 | handler.ServeHTTP(writer, r) 102 | }), 103 | ), 104 | ), 105 | ), 106 | ) 107 | return base.ServeHTTP 108 | } 109 | -------------------------------------------------------------------------------- /pkg/prealloc/prealloc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package prealloc 5 | 6 | import ( 7 | "bytes" 8 | "io" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/efficientgo/core/testutil" 13 | ) 14 | 15 | func withoutPrealloc(b testutil.TB) { 16 | const size = 1e6 17 | 18 | var slice []string 19 | for i := 0; i < size; i++ { 20 | slice = append(slice, "something") 21 | } 22 | 23 | var slice2 []string 24 | for i := 0; i < size; i++ { 25 | slice2 = append(slice2, "something") 26 | } 27 | 28 | m := make(map[int]string) 29 | for i := 0; i < size; i++ { 30 | m[i] = "something" 31 | } 32 | 33 | buf := bytes.Buffer{} 34 | for i := 0; i < size; i++ { 35 | _ = buf.WriteByte('a') 36 | } 37 | 38 | builder := strings.Builder{} 39 | for i := 0; i < size; i++ { 40 | builder.WriteByte('a') 41 | } 42 | 43 | buf2, _ := io.ReadAll(bytes.NewReader(make([]byte, size))) 44 | buf3, _ := io.ReadAll(bytes.NewReader(make([]byte, size))) 45 | 46 | // .Test 47 | if !b.IsBenchmark() { 48 | testutil.Equals(b, slice, slice2) 49 | testutil.Equals(b, buf.String(), builder.String()) 50 | testutil.Equals(b, buf2, buf3) 51 | testutil.Equals(b, buf2, make([]byte, size)) 52 | } 53 | } 54 | 55 | // Examples of allocations that can be pre-allocated. 56 | // Read more in "Efficient Go"; Example 11-11. 57 | func withPrealloc(b testutil.TB) { 58 | const size = 1e6 59 | 60 | slice := make([]string, 0, size) 61 | for i := 0; i < size; i++ { 62 | slice = append(slice, "something") 63 | } 64 | 65 | slice2 := make([]string, size) 66 | for i := 0; i < size; i++ { 67 | slice2[i] = "something" 68 | } 69 | 70 | m := make(map[int]string, size) 71 | for i := 0; i < size; i++ { 72 | m[i] = "something" 73 | } 74 | 75 | buf := bytes.Buffer{} 76 | buf.Grow(size) 77 | for i := 0; i < size; i++ { 78 | _ = buf.WriteByte('a') 79 | } 80 | 81 | builder := strings.Builder{} 82 | builder.Grow(size) 83 | for i := 0; i < size; i++ { 84 | builder.WriteByte('a') 85 | } 86 | 87 | buf2, _ := ReadAll1(bytes.NewReader(make([]byte, size)), size) 88 | buf3, _ := ReadAll2(bytes.NewReader(make([]byte, size)), size) 89 | 90 | // .Test 91 | if !b.IsBenchmark() { 92 | testutil.Equals(b, slice, slice2) 93 | testutil.Equals(b, buf.String(), builder.String()) 94 | testutil.Equals(b, buf2, buf3) 95 | testutil.Equals(b, buf2, make([]byte, size)) 96 | } 97 | } 98 | 99 | // Examples of pre-allocations for standard library helper `io.ReadAll`. 100 | // Read more in "Efficient Go"; Example 11-12. 101 | 102 | func ReadAll1(r io.Reader, size int) ([]byte, error) { 103 | buf := bytes.Buffer{} 104 | buf.Grow(size) 105 | n, err := io.Copy(&buf, r) 106 | return buf.Bytes()[:n], err 107 | } 108 | 109 | func ReadAll2(r io.Reader, size int) ([]byte, error) { 110 | buf := make([]byte, size) 111 | n, err := io.ReadFull(r, buf) 112 | if err == io.EOF { 113 | err = nil 114 | } 115 | return buf[:n], err 116 | } 117 | 118 | func TestAllocs(t *testing.T) { 119 | withoutPrealloc(testutil.NewTB(t)) 120 | withPrealloc(testutil.NewTB(t)) 121 | } 122 | 123 | func BenchmarkReadAlls(b *testing.B) { 124 | const size = int(1e6) 125 | inner := make([]byte, size) 126 | b.Run("io.ReadAll", func(b *testing.B) { 127 | b.ReportAllocs() 128 | for i := 0; i < b.N; i++ { 129 | buf, err := io.ReadAll(bytes.NewReader(inner)) 130 | testutil.Ok(b, err) 131 | testutil.Equals(b, size, len(buf)) 132 | } 133 | }) 134 | b.Run("ReadAll1", func(b *testing.B) { 135 | b.ReportAllocs() 136 | for i := 0; i < b.N; i++ { 137 | buf, err := ReadAll1(bytes.NewReader(inner), size) 138 | testutil.Ok(b, err) 139 | testutil.Equals(b, size, len(buf)) 140 | } 141 | }) 142 | b.Run("ReadAll2", func(b *testing.B) { 143 | b.ReportAllocs() 144 | for i := 0; i < b.N; i++ { 145 | buf, err := ReadAll2(bytes.NewReader(inner), size) 146 | testutil.Ok(b, err) 147 | testutil.Equals(b, size, len(buf)) 148 | } 149 | }) 150 | } 151 | -------------------------------------------------------------------------------- /pkg/prealloc/linkedlist_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package prealloc 5 | 6 | import ( 7 | "fmt" 8 | "runtime" 9 | "testing" 10 | 11 | "github.com/efficientgo/core/testutil" 12 | ) 13 | 14 | const size = 1e6 15 | 16 | func testLinkedList(t *testing.T, l *SinglyLinkedList) { 17 | t.Helper() 18 | 19 | for i := 0; i < size; i++ { 20 | l.Insert(i) 21 | } 22 | 23 | expected := make([]int, size) 24 | for i := 0; i < size; i++ { 25 | expected[i] = size - i - 1 26 | } 27 | 28 | got := make([]int, 0, size) 29 | for curr := l.head; curr != nil; curr = curr.next { 30 | got = append(got, curr.value) 31 | } 32 | testutil.Equals(t, expected, got) 33 | 34 | // Remove all but last. 35 | for curr := l.head; curr.next != nil; curr = curr.next { 36 | l.Delete(curr) 37 | } 38 | 39 | var got2 []int 40 | for curr := l.head; curr != nil; curr = curr.next { 41 | got2 = append(got2, curr.value) 42 | } 43 | testutil.Equals(t, []int{0}, got2) 44 | } 45 | 46 | func TestSinglyLinkedList(t *testing.T) { 47 | testLinkedList(t, &SinglyLinkedList{}) 48 | 49 | p := &SinglyLinkedList{} 50 | p.Grow(size) 51 | testLinkedList(t, p) 52 | } 53 | 54 | func BenchmarkSinglyLinkedList(b *testing.B) { 55 | const size = 1e6 56 | 57 | b.Run("normal", func(b *testing.B) { 58 | b.ReportAllocs() 59 | for i := 0; i < b.N; i++ { 60 | l := &SinglyLinkedList{} 61 | for k := 0; k < size; k++ { 62 | l.Insert(k) 63 | } 64 | } 65 | }) 66 | b.Run("pool", func(b *testing.B) { 67 | b.ReportAllocs() 68 | for i := 0; i < b.N; i++ { 69 | l := &SinglyLinkedList{} 70 | l.Grow(size) 71 | for k := 0; k < size; k++ { 72 | l.Insert(k) 73 | } 74 | } 75 | }) 76 | } 77 | 78 | func BenchmarkSinglyLinkedList_Delete(b *testing.B) { 79 | const size = 1e6 80 | 81 | b.Run("normal", func(b *testing.B) { 82 | b.ReportAllocs() 83 | for i := 0; i < b.N; i++ { 84 | b.StopTimer() 85 | l := &SinglyLinkedList{} 86 | for k := 0; k < size; k++ { 87 | l.Insert(k) 88 | } 89 | b.StartTimer() 90 | 91 | // Remove all but last. 92 | for curr := l.head; curr.next != nil; curr = curr.next { 93 | l.Delete(curr) 94 | } 95 | } 96 | }) 97 | b.Run("pool", func(b *testing.B) { 98 | b.ReportAllocs() 99 | for i := 0; i < b.N; i++ { 100 | b.StopTimer() 101 | l := &SinglyLinkedList{} 102 | l.Grow(size) 103 | for k := 0; k < size; k++ { 104 | l.Insert(k) 105 | } 106 | l.pool = nil // Dispose pool just in case. 107 | b.StartTimer() 108 | 109 | // Remove all but last. 110 | for curr := l.head; curr.next != nil; curr = curr.next { 111 | l.Delete(curr) 112 | } 113 | } 114 | }) 115 | } 116 | 117 | func _printHeapUsage(prefix string) { 118 | m := runtime.MemStats{} 119 | 120 | runtime.GC() 121 | runtime.ReadMemStats(&m) 122 | fmt.Println(prefix, float64(m.HeapAlloc)/1024.0, "KB") 123 | } 124 | 125 | func TestSinglyLinkedList_Delete1(t *testing.T) { 126 | l := &SinglyLinkedList{} 127 | for k := 0; k < size; k++ { 128 | l.Insert(k) 129 | } 130 | _printHeapUsage("Heap before deletions: ") 131 | 132 | // Remove all but last. 133 | for curr := l.head; curr.next != nil; curr = curr.next { 134 | l.Delete(curr) 135 | } 136 | _printHeapUsage("Heap after deleting all - 1: ") 137 | 138 | l.Delete(l.head) 139 | _printHeapUsage("Heap after last was deleted: ") 140 | } 141 | 142 | func TestSinglyLinkedList_Delete2(t *testing.T) { 143 | l := &SinglyLinkedList{} 144 | l.Grow(size) 145 | for k := 0; k < size; k++ { 146 | l.Insert(k) 147 | } 148 | l.pool = nil // Dispose pool. 149 | _printHeapUsage("Heap before deletions: ") 150 | 151 | // Remove all but last. 152 | for curr := l.head; curr.next != nil; curr = curr.next { 153 | l.Delete(curr) 154 | } 155 | _printHeapUsage("Heap after deleting all - 1: ") 156 | 157 | l.Delete(l.head) 158 | _printHeapUsage("Heap after last was deleted: ") 159 | } 160 | 161 | func TestSinglyLinkedList_Delete3(t *testing.T) { 162 | l := &SinglyLinkedList{} 163 | l.Grow(size) 164 | for k := 0; k < size; k++ { 165 | l.Insert(k) 166 | } 167 | l.pool = nil // Dispose pool. 168 | _printHeapUsage("Heap before deletions: ") 169 | 170 | // Remove all but last. 171 | for curr := l.head; curr.next != nil; curr = curr.next { 172 | l.Delete(curr) 173 | } 174 | _printHeapUsage("Heap after deleting all - 1: ") 175 | 176 | l.ClipMemory() 177 | 178 | _printHeapUsage("Heap after clipping: ") 179 | 180 | l.Delete(l.head) 181 | _printHeapUsage("Heap after last was deleted: ") 182 | } 183 | -------------------------------------------------------------------------------- /pkg/sum/labeler/labeler.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "crypto/sha256" 9 | "io" 10 | "os" 11 | "sync" 12 | 13 | "github.com/efficientgo/core/errcapture" 14 | "github.com/efficientgo/examples/pkg/profile/fd" 15 | "github.com/efficientgo/examples/pkg/sum" 16 | "github.com/gobwas/pool/pbytes" 17 | "github.com/thanos-io/objstore" 18 | ) 19 | 20 | type labelFunc func(ctx context.Context, objID string) (label, error) 21 | 22 | func bufferSize(fileSize int) int { 23 | s := fileSize / 64 24 | if s < 10e3 { 25 | return 10e3 26 | } 27 | return s 28 | } 29 | 30 | type labeler struct { 31 | bkt objstore.BucketReader 32 | 33 | tmpDir string 34 | pool sync.Pool 35 | bucketedPool *pbytes.Pool 36 | buf []byte 37 | } 38 | 39 | func (l *labeler) labelObject1(ctx context.Context, objID string) (_ label, err error) { 40 | a, err := l.bkt.Attributes(ctx, objID) 41 | if err != nil { 42 | return label{}, err 43 | } 44 | 45 | rc, err := l.bkt.Get(ctx, objID) 46 | if err != nil { 47 | return label{}, err 48 | } 49 | 50 | defer errcapture.Do(&err, rc.Close, "close stream") 51 | 52 | buf := make([]byte, bufferSize(int(a.Size))) 53 | s, err := sum.Sum6Reader(rc, buf) 54 | if err != nil { 55 | return label{}, err 56 | } 57 | 58 | // Get/calculate other attributes... 59 | 60 | return label{ 61 | ObjID: objID, 62 | Sum: s, 63 | // ... 64 | }, nil 65 | } 66 | 67 | func (l *labeler) labelObjectNaive(ctx context.Context, objID string) (_ label, err error) { 68 | rc, err := l.bkt.Get(ctx, objID) 69 | if err != nil { 70 | return label{}, err 71 | } 72 | 73 | // Download file first. 74 | // TODO(bwplotka): This is naive for book purposes. 75 | f, err := fd.CreateTemp(l.tmpDir, "cached-*") 76 | if err != nil { 77 | return label{}, err 78 | } 79 | defer func() { 80 | _ = f.Close() 81 | _ = os.RemoveAll(f.Name()) 82 | }() 83 | 84 | h := sha256.New() 85 | 86 | // Write to both checksum hash and file. 87 | if _, err := io.Copy(f, io.TeeReader(rc, h)); err != nil { 88 | return label{}, err 89 | } 90 | if err := rc.Close(); err != nil { 91 | return label{}, err 92 | } 93 | 94 | s, err := sum.Sum(f.Name()) 95 | if err != nil { 96 | return label{}, err 97 | } 98 | 99 | // Get/calculate other attributes... 100 | 101 | return label{ 102 | ObjID: objID, 103 | Sum: s, 104 | CheckSum: h.Sum(nil), 105 | // ... 106 | }, nil 107 | } 108 | 109 | func (l *labeler) labelObject2(ctx context.Context, objID string) (_ label, err error) { 110 | a, err := l.bkt.Attributes(ctx, objID) 111 | if err != nil { 112 | return label{}, err 113 | } 114 | 115 | rc, err := l.bkt.Get(ctx, objID) 116 | if err != nil { 117 | return label{}, err 118 | } 119 | 120 | defer errcapture.Do(&err, rc.Close, "close stream") 121 | 122 | bufSize := bufferSize(int(a.Size)) 123 | buf := l.pool.Get().([]byte) 124 | if cap(buf) < bufSize { 125 | buf = make([]byte, bufSize) 126 | } 127 | defer func() { l.pool.Put(buf) }() 128 | 129 | s, err := sum.Sum6Reader(rc, buf[:bufSize]) 130 | if err != nil { 131 | return label{}, err 132 | } 133 | 134 | // Get/calculate other attributes... 135 | 136 | return label{ 137 | ObjID: objID, 138 | Sum: s, 139 | // ... 140 | }, nil 141 | } 142 | 143 | func (l *labeler) labelObject3(ctx context.Context, objID string) (_ label, err error) { 144 | a, err := l.bkt.Attributes(ctx, objID) 145 | if err != nil { 146 | return label{}, err 147 | } 148 | 149 | rc, err := l.bkt.Get(ctx, objID) 150 | if err != nil { 151 | return label{}, err 152 | } 153 | 154 | defer errcapture.Do(&err, rc.Close, "close stream") 155 | 156 | bufSize := bufferSize(int(a.Size)) 157 | buf := l.bucketedPool.Get(bufSize, bufSize) 158 | if cap(buf) < bufSize { 159 | buf = make([]byte, bufSize) 160 | } 161 | defer func() { l.bucketedPool.Put(buf) }() 162 | 163 | s, err := sum.Sum6Reader(rc, buf[:bufSize]) 164 | if err != nil { 165 | return label{}, err 166 | } 167 | 168 | // Get/calculate other attributes... 169 | 170 | return label{ 171 | ObjID: objID, 172 | Sum: s, 173 | // ... 174 | }, nil 175 | } 176 | 177 | func (l *labeler) labelObject4(ctx context.Context, objID string) (_ label, err error) { 178 | a, err := l.bkt.Attributes(ctx, objID) 179 | if err != nil { 180 | return label{}, err 181 | } 182 | 183 | rc, err := l.bkt.Get(ctx, objID) 184 | if err != nil { 185 | return label{}, err 186 | } 187 | 188 | defer errcapture.Do(&err, rc.Close, "close stream") 189 | 190 | bufSize := bufferSize(int(a.Size)) 191 | if cap(l.buf) < bufSize { 192 | l.buf = make([]byte, bufSize) 193 | } 194 | s, err := sum.Sum6Reader(rc, l.buf[:bufSize]) 195 | if err != nil { 196 | return label{}, err 197 | } 198 | 199 | // Get/calculate other attributes... 200 | 201 | return label{ 202 | ObjID: objID, 203 | Sum: s, 204 | // ... 205 | }, nil 206 | } 207 | -------------------------------------------------------------------------------- /pkg/sum/labeler/labeler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "runtime" 10 | "sync" 11 | "testing" 12 | 13 | "github.com/efficientgo/core/testutil" 14 | "github.com/efficientgo/examples/pkg/sum/sumtestutil" 15 | "github.com/gobwas/pool/pbytes" 16 | "github.com/thanos-io/objstore" 17 | ) 18 | 19 | func bench1(b *testing.B, labelFn func(ctx context.Context, objID string) (label, error)) { 20 | b.ReportAllocs() 21 | 22 | ctx := context.Background() 23 | var err error 24 | 25 | b.ResetTimer() 26 | for i := 0; i < b.N; i++ { 27 | if i%2 == 0 { 28 | _, err = labelFn(ctx, "10M.txt") 29 | testutil.Ok(b, err) 30 | continue 31 | } 32 | _, err = labelFn(ctx, "100M.txt") 33 | testutil.Ok(b, err) 34 | } 35 | } 36 | 37 | func bench2(b *testing.B, labelFn func(ctx context.Context, objID string) (label, error)) { 38 | b.ReportAllocs() 39 | 40 | ctx := context.Background() 41 | var err error 42 | 43 | wg := sync.WaitGroup{} 44 | wg.Add(4) 45 | 46 | b.ResetTimer() 47 | for g := 0; g < 4; g++ { 48 | go func() { 49 | defer wg.Done() 50 | 51 | for i := 0; i < b.N; i++ { 52 | _, err = labelFn(ctx, "10M.txt") 53 | testutil.Ok(b, err) 54 | _, err = labelFn(ctx, "100M.txt") 55 | testutil.Ok(b, err) 56 | runtime.GC() 57 | } 58 | }() 59 | } 60 | wg.Wait() 61 | 62 | } 63 | 64 | // BenchmarkLabeler recommended run options: 65 | // $ export ver=v1 && go test -run '^$' -bench '^BenchmarkLabeler' -benchtime 100x -count 6 -cpu 1 -benchmem -memprofile=${ver}.mem.pprof -cpuprofile=${ver}.cpu.pprof | tee ${ver}.txt 66 | func BenchmarkLabeler(b *testing.B) { 67 | ctx := context.Background() 68 | bkt := objstore.NewInMemBucket() 69 | 70 | buf := bytes.Buffer{} 71 | _, err := sumtestutil.CreateTestInputWithExpectedResult(&buf, 1e7) 72 | testutil.Ok(b, err) 73 | testutil.Ok(b, bkt.Upload(ctx, "10M.txt", &buf)) 74 | 75 | buf.Reset() 76 | _, err = sumtestutil.CreateTestInputWithExpectedResult(&buf, 1e8) 77 | testutil.Ok(b, err) 78 | testutil.Ok(b, bkt.Upload(ctx, "100M.txt", &buf)) 79 | 80 | b.Run("labelObject1", func(b *testing.B) { 81 | l := &labeler{bkt: bkt} 82 | 83 | bench1(b, l.labelObject1) 84 | }) 85 | b.Run("labelObject2", func(b *testing.B) { 86 | l := &labeler{bkt: bkt} 87 | l.pool.New = func() any { return []byte(nil) } 88 | 89 | bench1(b, l.labelObject2) 90 | }) 91 | b.Run("labelObject3", func(b *testing.B) { 92 | l := &labeler{bkt: bkt} 93 | 94 | l.bucketedPool = pbytes.New(1e3, 10e6) 95 | bench1(b, l.labelObject3) 96 | }) 97 | b.Run("labelObject4", func(b *testing.B) { 98 | l := &labeler{bkt: bkt} 99 | 100 | bench1(b, l.labelObject4) 101 | }) 102 | } 103 | 104 | func TestLabeler(t *testing.T) { 105 | ctx := context.Background() 106 | bkt := objstore.NewInMemBucket() 107 | 108 | buf := bytes.Buffer{} 109 | exp1, err := sumtestutil.CreateTestInputWithExpectedResult(&buf, 2e6) 110 | testutil.Ok(t, err) 111 | testutil.Ok(t, bkt.Upload(ctx, "2M.txt", &buf)) 112 | 113 | buf.Reset() 114 | exp2, err := sumtestutil.CreateTestInputWithExpectedResult(&buf, 1e7) 115 | testutil.Ok(t, err) 116 | testutil.Ok(t, bkt.Upload(ctx, "100M.txt", &buf)) 117 | 118 | t.Run("labelObjectNaive", func(t *testing.T) { 119 | l := &labeler{bkt: bkt} 120 | l.tmpDir = t.TempDir() 121 | 122 | ret, err := l.labelObject1(ctx, "2M.txt") 123 | testutil.Ok(t, err) 124 | testutil.Equals(t, exp1, ret.Sum) 125 | ret, err = l.labelObject1(ctx, "100M.txt") 126 | testutil.Ok(t, err) 127 | testutil.Equals(t, exp2, ret.Sum) 128 | }) 129 | 130 | t.Run("labelObject1", func(t *testing.T) { 131 | l := &labeler{bkt: bkt} 132 | 133 | ret, err := l.labelObject1(ctx, "2M.txt") 134 | testutil.Ok(t, err) 135 | testutil.Equals(t, exp1, ret.Sum) 136 | ret, err = l.labelObject1(ctx, "100M.txt") 137 | testutil.Ok(t, err) 138 | testutil.Equals(t, exp2, ret.Sum) 139 | }) 140 | t.Run("labelObject2", func(t *testing.T) { 141 | l := &labeler{bkt: bkt} 142 | l.pool.New = func() any { return []byte(nil) } 143 | 144 | ret, err := l.labelObject2(ctx, "2M.txt") 145 | testutil.Ok(t, err) 146 | testutil.Equals(t, exp1, ret.Sum) 147 | ret, err = l.labelObject2(ctx, "100M.txt") 148 | testutil.Ok(t, err) 149 | testutil.Equals(t, exp2, ret.Sum) 150 | }) 151 | t.Run("labelObject3", func(t *testing.T) { 152 | l := &labeler{bkt: bkt} 153 | 154 | l.bucketedPool = pbytes.New(1e3, 10e6) 155 | 156 | ret, err := l.labelObject3(ctx, "2M.txt") 157 | testutil.Ok(t, err) 158 | testutil.Equals(t, exp1, ret.Sum) 159 | ret, err = l.labelObject3(ctx, "100M.txt") 160 | testutil.Ok(t, err) 161 | testutil.Equals(t, exp2, ret.Sum) 162 | }) 163 | t.Run("labelObject4", func(t *testing.T) { 164 | l := &labeler{bkt: bkt} 165 | 166 | l.buf = make([]byte, 10e3) 167 | ret, err := l.labelObject4(ctx, "2M.txt") 168 | testutil.Ok(t, err) 169 | testutil.Equals(t, exp1, ret.Sum) 170 | ret, err = l.labelObject4(ctx, "100M.txt") 171 | testutil.Ok(t, err) 172 | testutil.Equals(t, exp2, ret.Sum) 173 | }) 174 | } 175 | -------------------------------------------------------------------------------- /pkg/sum/labeler/labeler_e2e_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "fmt" 10 | "os" 11 | "strconv" 12 | "testing" 13 | 14 | "github.com/efficientgo/core/testutil" 15 | "github.com/efficientgo/e2e" 16 | e2edb "github.com/efficientgo/e2e/db" 17 | e2einteractive "github.com/efficientgo/e2e/interactive" 18 | e2emonitoring "github.com/efficientgo/e2e/monitoring" 19 | "github.com/efficientgo/examples/pkg/sum/sumtestutil" 20 | "github.com/go-kit/log" 21 | "github.com/thanos-io/objstore/client" 22 | "github.com/thanos-io/objstore/providers/s3" 23 | "gopkg.in/yaml.v3" 24 | ) 25 | 26 | const bktName = "test" 27 | 28 | func marshal(t testing.TB, i interface{}) string { 29 | t.Helper() 30 | 31 | b, err := yaml.Marshal(i) 32 | testutil.Ok(t, err) 33 | 34 | return string(b) 35 | } 36 | 37 | // TestLabeler_LabelObject runs interactive macro benchmark for `labeler` program. 38 | // Prerequisites: 39 | // * `docker` CLI and docker engine installed. 40 | // * Run `make docker` from root project to build `labeler:latest` docker image. 41 | // Read more in "Efficient Go"; Example 8-19, 8-20, 42 | func TestLabeler_LabelObject(t *testing.T) { 43 | t.Skip("Comment this line if you want to run it - it's interactive test. Won't be useful in CI") 44 | 45 | e, err := e2e.NewDockerEnvironment("labeler") 46 | testutil.Ok(t, err) 47 | t.Cleanup(e.Close) 48 | 49 | // Start monitoring. 50 | mon, err := e2emonitoring.Start(e) 51 | testutil.Ok(t, err) 52 | testutil.Ok(t, mon.OpenUserInterfaceInBrowser()) 53 | 54 | // Start storage. 55 | minio := e2edb.NewMinio(e, "object-storage", bktName) 56 | testutil.Ok(t, e2e.StartAndWaitReady(minio)) 57 | 58 | // Run program we want to test and benchmark. 59 | labeler := e2e.NewInstrumentedRunnable(e, "labeler"). 60 | WithPorts(map[string]int{"http": 8080}, "http"). 61 | Init(e2e.StartOptions{ 62 | Image: "labeler:test", 63 | LimitCPUs: 4.0, 64 | Command: e2e.NewCommand( 65 | "/labeler", 66 | "-listen-address=:8080", 67 | "-objstore.config="+marshal(t, client.BucketConfig{ 68 | Type: client.S3, 69 | Config: s3.Config{ 70 | Bucket: bktName, 71 | AccessKey: e2edb.MinioAccessKey, 72 | SecretKey: e2edb.MinioSecretKey, 73 | Endpoint: minio.InternalEndpoint(e2edb.AccessPortName), 74 | Insecure: true, 75 | }, 76 | }), 77 | ), 78 | }) 79 | testutil.Ok(t, e2e.StartAndWaitReady(labeler)) 80 | 81 | // Add test file. 82 | testutil.Ok(t, uploadTestInput(minio, "object1.txt", 2e6)) 83 | 84 | // Start continuous profiling (not present in examples 8-19, 8-20). 85 | parca := e2e.NewInstrumentedRunnable(e, "parca"). 86 | WithPorts(map[string]int{"http": 7070}, "http"). 87 | Init(e2e.StartOptions{ 88 | Image: "ghcr.io/parca-dev/parca:main-4e20a666", 89 | Command: e2e.NewCommand("/bin/sh", "-c", 90 | `cat << EOF > /shared/data/config.yml && \ 91 | /parca --config-path=/shared/data/config.yml 92 | object_storage: 93 | bucket: 94 | type: "FILESYSTEM" 95 | config: 96 | directory: "./data" 97 | scrape_configs: 98 | - job_name: "labeler" 99 | scrape_interval: "15s" 100 | static_configs: 101 | - targets: [ '`+labeler.InternalEndpoint("http")+`' ] 102 | profiling_config: 103 | pprof_config: 104 | fgprof: 105 | enabled: true 106 | path: /debug/fgprof/profile 107 | delta: true 108 | EOF 109 | `), 110 | User: strconv.Itoa(os.Getuid()), 111 | Readiness: e2e.NewTCPReadinessProbe("http"), 112 | }) 113 | testutil.Ok(t, e2e.StartAndWaitReady(parca)) 114 | testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+parca.Endpoint("http"))) 115 | 116 | // Load test labeler from 1 clients with k6 and export result to Prometheus. 117 | k6 := e.Runnable("k6").Init(e2e.StartOptions{ 118 | Command: e2e.NewCommandRunUntilStop(), 119 | Image: "grafana/k6:0.39.0", 120 | }) 121 | testutil.Ok(t, e2e.StartAndWaitReady(k6)) 122 | 123 | url := fmt.Sprintf( 124 | "http://%s/label_object?object_id=object1.txt", 125 | labeler.InternalEndpoint("http"), 126 | ) 127 | testutil.Ok(t, k6.Exec(e2e.NewCommand( 128 | "/bin/sh", "-c", 129 | `cat << EOF | k6 run -u 1 -d 5m - 130 | import http from 'k6/http'; 131 | import { check, sleep } from 'k6'; 132 | 133 | export default function () { 134 | const res = http.get('`+url+`'); 135 | check(res, { 136 | 'is status 200': (r) => r.status === 200, 137 | 'response': (r) => 138 | r.body.includes('{"object_id":"object1.txt","sum":6221600000,"checksum":"SUUr'), 139 | }); 140 | sleep(0.5) 141 | } 142 | EOF`))) 143 | 144 | // Once done, wait for user input so user can explore the results in Prometheus UI and logs. 145 | testutil.Ok(t, e2einteractive.RunUntilEndpointHit()) 146 | } 147 | 148 | func uploadTestInput(m e2e.Runnable, objID string, numLen int) error { 149 | bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{ 150 | Bucket: bktName, 151 | AccessKey: e2edb.MinioAccessKey, 152 | SecretKey: e2edb.MinioSecretKey, 153 | Endpoint: m.Endpoint(e2edb.AccessPortName), 154 | Insecure: true, 155 | }, "test") 156 | if err != nil { 157 | return err 158 | } 159 | 160 | b := bytes.Buffer{} 161 | if _, err := sumtestutil.CreateTestInputWithExpectedResult(&b, numLen); err != nil { 162 | return err 163 | } 164 | 165 | return bkt.Upload(context.Background(), objID, &b) 166 | } 167 | -------------------------------------------------------------------------------- /pkg/metrics/mem_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package metrics 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "regexp" 11 | "runtime" 12 | "runtime/metrics" 13 | "testing" 14 | 15 | "github.com/prometheus/client_golang/prometheus" 16 | "github.com/prometheus/client_golang/prometheus/collectors" 17 | "github.com/prometheus/client_golang/prometheus/promhttp" 18 | ) 19 | 20 | func naivePrintMemStats() { 21 | runtime.GC() 22 | mem := runtime.MemStats{} 23 | runtime.ReadMemStats(&mem) 24 | // Example output: 2022/04/09 13:42:33 { 25 | // Alloc:472536 26 | // TotalAlloc:773208 27 | // Sys:11027464 28 | // Lookups:0 29 | // Mallocs:3543 30 | // Frees:1929 31 | // HeapAlloc:472536 32 | // HeapSys:3735552 33 | // HeapIdle:2170880 34 | // HeapInuse:1564672 35 | // HeapReleased:1720320 36 | // HeapObjects:1614 37 | // StackInuse:458752 38 | // StackSys:458752 39 | // MSpanInuse:55080 40 | // MSpanSys:65536 41 | // MCacheInuse:14400 42 | // MCacheSys:16384 43 | // BuckHashSys:1445701 44 | // GCSys:4009176 45 | // OtherSys:1296363 46 | // NextGC:4194304 47 | // LastGC:1649508153943084326 48 | // PauseTotalNs:15783 49 | // PauseNs:[15783 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] 50 | // PauseEnd:[1649508153943084326 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] 51 | // NumGC:1 52 | // NumForcedGC:1 53 | // GCCPUFraction:0.015881668262409922 54 | // EnableGC:true 55 | // DebugGC:false 56 | // BySize:[{Size:0 Mallocs:0 Frees:0} {Size:8 Mallocs:83 Frees:37} {Size:16 Mallocs:897 Frees:457} {Size:24 Mallocs:508 Frees:431} {Size:32 Mallocs:307 Frees:155} {Size:48 Mallocs:302 Frees:80} {Size:64 Mallocs:149 Frees:47} {Size:80 Mallocs:62 Frees:51} {Size:96 Mallocs:88 Frees:35} {Size:112 Mallocs:443 Frees:260} {Size:128 Mallocs:27 Frees:14} {Size:144 Mallocs:0 Frees:0} {Size:160 Mallocs:79 Frees:36} {Size:176 Mallocs:25 Frees:3} {Size:192 Mallocs:1 Frees:0} {Size:208 Mallocs:60 Frees:34} {Size:224 Mallocs:1 Frees:0} {Size:240 Mallocs:1 Frees:0} {Size:256 Mallocs:23 Frees:6} {Size:288 Mallocs:14 Frees:5} {Size:320 Mallocs:35 Frees:26} {Size:352 Mallocs:25 Frees:6} {Size:384 Mallocs:2 Frees:1} {Size:416 Mallocs:60 Frees:15} {Size:448 Mallocs:12 Frees:0} {Size:480 Mallocs:3 Frees:0} {Size:512 Mallocs:2 Frees:2} {Size:576 Mallocs:11 Frees:7} {Size:640 Mallocs:29 Frees:15} {Size:704 Mallocs:8 Frees:3} {Size:768 Mallocs:0 Frees:0} {Size:896 Mallocs:15 Frees:11} {Size:1024 Mallocs:13 Frees:2} {Size:1152 Mallocs:6 Frees:3} {Size:1280 Mallocs:16 Frees:7} {Size:1408 Mallocs:5 Frees:3} {Size:1536 Mallocs:7 Frees:2} {Size:1792 Mallocs:15 Frees:6} {Size:2048 Mallocs:1 Frees:0} {Size:2304 Mallocs:6 Frees:0} {Size:2688 Mallocs:8 Frees:3} {Size:3072 Mallocs:0 Frees:0} {Size:3200 Mallocs:1 Frees:0} {Size:3456 Mallocs:0 Frees:0} {Size:4096 Mallocs:8 Frees:4} {Size:4864 Mallocs:3 Frees:3} {Size:5376 Mallocs:2 Frees:0} {Size:6144 Mallocs:7 Frees:4} {Size:6528 Mallocs:0 Frees:0} {Size:6784 Mallocs:0 Frees:0} {Size:6912 Mallocs:0 Frees:0} {Size:8192 Mallocs:2 Frees:0} {Size:9472 Mallocs:2 Frees:0} {Size:9728 Mallocs:0 Frees:0} {Size:10240 Mallocs:12 Frees:0} {Size:10880 Mallocs:1 Frees:1} {Size:12288 Mallocs:0 Frees:0} {Size:13568 Mallocs:1 Frees:0} {Size:14336 Mallocs:0 Frees:0} {Size:16384 Mallocs:0 Frees:0} {Size:18432 Mallocs:0 Frees:0}]} 57 | log.Printf("%+v\n", mem) 58 | } 59 | 60 | var memMetrics = []metrics.Sample{ 61 | // Total bytes allocated. 62 | {Name: "/gc/heap/allocs:bytes"}, 63 | // Currently used bytes on heap. 64 | {Name: "/memory/classes/heap/objects:bytes"}, 65 | } 66 | 67 | // Example of getting CPU usage through Prometheus metrics 68 | // Read more in "Efficient Go"; Example 6-12. 69 | func printMemRuntimeMetric() { 70 | runtime.GC() 71 | metrics.Read(memMetrics) 72 | 73 | fmt.Println("Total bytes allocaed:", memMetrics[0].Value.Uint64()) 74 | fmt.Println("Current inuse bytes:", memMetrics[1].Value.Uint64()) 75 | } 76 | 77 | func TestRTMetrics(t *testing.T) { 78 | printMemRuntimeMetric() 79 | naivePrintMemStats() 80 | } 81 | 82 | func ExampleMemoryMetrics() { 83 | reg := prometheus.NewRegistry() 84 | reg.MustRegister(collectors.NewGoCollector( 85 | collectors.WithGoCollectorRuntimeMetrics( 86 | collectors.GoRuntimeMetricsRule{ 87 | Matcher: regexp.MustCompile("/gc/heap/allocs:bytes"), 88 | }, 89 | collectors.GoRuntimeMetricsRule{ 90 | Matcher: regexp.MustCompile("/memory/classes/heap/objects:bytes"), 91 | }, 92 | ))) 93 | 94 | go http.ListenAndServe(":8080", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) 95 | 96 | for i := 0; i < xTimes; i++ { 97 | err := doOperation() 98 | // ... 99 | _ = err 100 | } 101 | 102 | printPrometheusMetrics(reg) 103 | } 104 | -------------------------------------------------------------------------------- /pkg/sum/sum_concurrent.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package sum 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "os" 12 | "sync" 13 | "sync/atomic" 14 | 15 | "github.com/efficientgo/core/errcapture" 16 | "github.com/efficientgo/core/errors" 17 | ) 18 | 19 | // ConcurrentSum1 performs sum concurrently. A lot slower than ConcurrentSum3. An example of pessimisation. 20 | // Read more in "Efficient Go"; Example 10-10. 21 | func ConcurrentSum1(fileName string) (ret int64, _ error) { 22 | b, err := ioutil.ReadFile(fileName) 23 | if err != nil { 24 | return 0, err 25 | } 26 | 27 | var wg sync.WaitGroup 28 | var last int 29 | for i := 0; i < len(b); i++ { 30 | if b[i] != '\n' { 31 | continue 32 | } 33 | 34 | wg.Add(1) 35 | go func(line []byte) { // Creation of goroutine turns to be mem intensive on scale! (on top of time) 36 | defer wg.Done() 37 | num, err := ParseInt(line) 38 | if err != nil { 39 | // TODO(bwplotka): Return err using other channel. 40 | return 41 | } 42 | atomic.AddInt64(&ret, num) 43 | }(b[last:i]) 44 | last = i + 1 45 | } 46 | wg.Wait() 47 | return ret, nil 48 | } 49 | 50 | // ConcurrentSum2 performs sum concurrently. A lot slower than ConcurrentSum3. An example of pessimisation. 51 | // Read more in "Efficient Go"; Example 10-11. 52 | func ConcurrentSum2(fileName string, workers int) (ret int64, _ error) { 53 | b, err := ioutil.ReadFile(fileName) 54 | if err != nil { 55 | return 0, err 56 | } 57 | 58 | var ( 59 | wg = sync.WaitGroup{} 60 | workCh = make(chan []byte, workers) 61 | ) 62 | 63 | wg.Add(workers + 1) 64 | go func() { 65 | var last int 66 | for i := 0; i < len(b); i++ { 67 | if b[i] != '\n' { 68 | continue 69 | } 70 | workCh <- b[last:i] 71 | last = i + 1 72 | } 73 | close(workCh) 74 | wg.Done() 75 | }() 76 | 77 | for i := 0; i < workers; i++ { 78 | go func() { 79 | var sum int64 80 | for line := range workCh { // Common mistake: for _, line := range <-workCh 81 | num, err := ParseInt(line) 82 | if err != nil { 83 | // TODO(bwplotka): Return err using other channel. 84 | continue 85 | } 86 | sum += num 87 | } 88 | atomic.AddInt64(&ret, sum) 89 | wg.Done() 90 | }() 91 | } 92 | wg.Wait() 93 | return ret, nil 94 | } 95 | 96 | // Over inline budget, but for readability it's better. Consider splitting functions if needed to get it inlinded. 97 | //./sum_concurrent.go:11:6: cannot inline shardedRange: function too complex: cost 95 exceeds budget 80 98 | func shardedRange(routineNumber int, bytesPerWorker int, b []byte) (int, int) { 99 | begin := routineNumber * bytesPerWorker 100 | end := begin + bytesPerWorker 101 | if end+bytesPerWorker > len(b) { 102 | end = len(b) 103 | } 104 | 105 | // Find last newline before begin and add 1. If not found (-1), it means we 106 | // are at the start. Otherwise, we start after last newline. 107 | return bytes.LastIndex(b[:begin], []byte("\n")) + 1, end 108 | } 109 | 110 | // ConcurrentSum3 uses coordination free sharding to perform more efficient computation. 111 | // Read more in "Efficient Go"; Example 10-12. 112 | func ConcurrentSum3(fileName string, workers int) (ret int64, _ error) { 113 | b, err := ioutil.ReadFile(fileName) 114 | if err != nil { 115 | return 0, err 116 | } 117 | 118 | var ( 119 | bytesPerWorker = len(b) / workers 120 | resultCh = make(chan int64) 121 | ) 122 | 123 | for i := 0; i < workers; i++ { 124 | go func(i int) { 125 | // Coordination-free algorithm, which shards buffered file deterministically. 126 | begin, end := shardedRange(i, bytesPerWorker, b) 127 | 128 | var sum int64 129 | for last := begin; begin < end; begin++ { 130 | if b[begin] != '\n' { 131 | continue 132 | } 133 | num, err := ParseInt(b[last:begin]) 134 | if err != nil { 135 | // TODO(bwplotka): Return err using other channel. 136 | continue 137 | } 138 | sum += num 139 | last = begin + 1 140 | } 141 | resultCh <- sum 142 | }(i) 143 | } 144 | 145 | for i := 0; i < workers; i++ { 146 | ret += <-resultCh 147 | } 148 | close(resultCh) 149 | return ret, nil 150 | } 151 | 152 | func shardedRangeFromReaderAt(routineNumber int, bytesPerWorker int, size int, f io.ReaderAt) (int, int) { 153 | begin := routineNumber * bytesPerWorker 154 | end := begin + bytesPerWorker 155 | if end+bytesPerWorker > size { 156 | end = size 157 | } 158 | 159 | if begin == 0 { 160 | return begin, end 161 | } 162 | 163 | const maxNumSize = 10 164 | buf := make([]byte, maxNumSize) 165 | begin -= maxNumSize 166 | 167 | if _, err := f.ReadAt(buf, int64(begin)); err != nil { 168 | // TODO(bwplotka): Return err using other channel. 169 | fmt.Println(err) 170 | return 0, 0 171 | } 172 | 173 | for i := maxNumSize; i > 0; i-- { 174 | if buf[i-1] == '\n' { 175 | begin += i 176 | break 177 | } 178 | } 179 | return begin, end 180 | } 181 | 182 | // ConcurrentSum4 is like ConcurrentSum3, but it reads file in sharded way too. 183 | // Read more in "Efficient Go"; Example 10-13. 184 | func ConcurrentSum4(fileName string, workers int) (ret int64, _ error) { 185 | f, err := os.Open(fileName) 186 | if err != nil { 187 | return 0, err 188 | } 189 | defer errcapture.Do(&err, f.Close, "close file") 190 | 191 | s, err := f.Stat() 192 | if err != nil { 193 | return 0, err 194 | } 195 | 196 | var ( 197 | size = int(s.Size()) 198 | bytesPerWorker = size / workers 199 | resultCh = make(chan int64) 200 | ) 201 | 202 | if bytesPerWorker < 10 { 203 | return 0, errors.New("can't have less bytes per goroutine than 10") 204 | } 205 | 206 | for i := 0; i < workers; i++ { 207 | go func(i int) { 208 | begin, end := shardedRangeFromReaderAt(i, bytesPerWorker, size, f) 209 | r := io.NewSectionReader(f, int64(begin), int64(end-begin)) 210 | 211 | b := make([]byte, 8*1024) 212 | sum, err := Sum6Reader(r, b) 213 | if err != nil { 214 | // TODO(bwplotka): Return err using other channel. 215 | fmt.Println(err) 216 | } 217 | resultCh <- sum 218 | }(i) 219 | } 220 | 221 | for i := 0; i < workers; i++ { 222 | ret += <-resultCh 223 | } 224 | close(resultCh) 225 | return ret, nil 226 | } 227 | -------------------------------------------------------------------------------- /pkg/sum/sum_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package sum 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | "unsafe" 12 | 13 | "github.com/efficientgo/core/errcapture" 14 | "github.com/efficientgo/core/errors" 15 | "github.com/efficientgo/core/testutil" 16 | "github.com/efficientgo/examples/pkg/sum/sumtestutil" 17 | "github.com/felixge/fgprof" 18 | ) 19 | 20 | func createTestInput(fn string, numLen int) error { 21 | _, err := createTestInputWithExpectedResult(fn, numLen) 22 | return err 23 | } 24 | 25 | // lazyCreateTestInput creates test input on the fly if not cached. 26 | // Read more in "Efficient Go"; Example 8-13. 27 | func lazyCreateTestInput(tb testing.TB, numLines int) string { 28 | tb.Helper() 29 | 30 | fn := fmt.Sprintf("testdata/test.%v.txt", numLines) 31 | if _, err := os.Stat(fn); errors.Is(err, os.ErrNotExist) { 32 | testutil.Ok(tb, createTestInput(fn, numLines)) 33 | } else { 34 | testutil.Ok(tb, err) 35 | } 36 | return fn 37 | } 38 | 39 | // BenchmarkSum benchmarks `Sum` function. 40 | // NOTE(bwplotka): Test it with maximum of 4 CPU cores, given we don't allocate 41 | // more in our production containers. 42 | // 43 | // Recommended run options: 44 | /* 45 | export ver=v1 && go test \ 46 | -run '^$' -bench '^BenchmarkSum$' \ 47 | -benchtime 10s -count 6 -cpu 4 -benchmem \ 48 | -memprofile=${ver}.mem.pprof -cpuprofile=${ver}.cpu.pprof \ 49 | | tee ${ver}.txt 50 | */ 51 | // Read more in "Efficient Go"; Example 8-1, 8-2, 8-10, 8-12 52 | func BenchmarkSum(b *testing.B) { 53 | fn := lazyCreateTestInput(b, 2e6) 54 | 55 | b.ResetTimer() 56 | for i := 0; i < b.N; i++ { 57 | _, err := Sum(fn) 58 | testutil.Ok(b, err) 59 | } 60 | } 61 | 62 | func createTestInputWithExpectedResult(fn string, numLen int) (sum int64, err error) { 63 | if err := os.MkdirAll(filepath.Dir(fn), os.ModePerm); err != nil { 64 | return 0, err 65 | } 66 | 67 | f, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY, os.ModePerm) 68 | if err != nil { 69 | return 0, errors.Wrap(err, "open") 70 | } 71 | 72 | defer errcapture.Do(&err, f.Close, "close file") 73 | 74 | return sumtestutil.CreateTestInputWithExpectedResult(f, numLen) 75 | } 76 | 77 | // TestSum tests all sum implementations. 78 | // Read more in "Efficient Go"; Example 8-9. 79 | func TestSum(t *testing.T) { 80 | testFile := filepath.Join(t.TempDir(), "input.txt") 81 | expectedSum, err := createTestInputWithExpectedResult(testFile, 2e6) 82 | testutil.Ok(t, err) 83 | 84 | t.Run("simple", func(t *testing.T) { 85 | for _, tcase := range []struct { 86 | f func(string) (int64, error) 87 | }{ 88 | {f: Sum}, {f: Sum2}, {f: Sum2_scanner}, {f: ConcurrentSum1}, {f: Sum3}, 89 | {f: Sum4}, {f: Sum4_atoi}, {f: Sum5}, {f: Sum5_line}, {f: Sum6}, {f: Sum7}, 90 | } { 91 | t.Run("", func(t *testing.T) { 92 | ret, err := tcase.f(testFile) 93 | testutil.Ok(t, err) 94 | testutil.Equals(t, expectedSum, ret) 95 | }) 96 | } 97 | }) 98 | t.Run("workers", func(t *testing.T) { 99 | for _, tcase := range []struct { 100 | f func(string, int) (int64, error) 101 | }{ 102 | {f: ConcurrentSum2}, {f: ConcurrentSum3}, {f: ConcurrentSum4}, 103 | } { 104 | t.Run("", func(t *testing.T) { 105 | ret, err := tcase.f(testFile, 4) 106 | testutil.Ok(t, err) 107 | testutil.Equals(t, expectedSum, ret) 108 | 109 | ret, err = tcase.f(testFile, 11) 110 | testutil.Ok(t, err) 111 | testutil.Equals(t, expectedSum, ret) 112 | }) 113 | } 114 | }) 115 | } 116 | 117 | // TestBenchSum tests the benchmark (!). 118 | // Read more in "Efficient Go"; Example 8-11. 119 | func TestBenchSum(t *testing.T) { 120 | benchmarkSum(testutil.NewTB(t)) 121 | } 122 | 123 | func BenchmarkSum_tested(b *testing.B) { 124 | benchmarkSum(testutil.NewTB(b)) 125 | } 126 | 127 | func benchmarkSum(tb testutil.TB) { 128 | fn := lazyCreateTestInput(tb, 2e6) 129 | 130 | tb.ResetTimer() 131 | for i := 0; i < tb.N(); i++ { 132 | ret, err := Sum(fn) 133 | testutil.Ok(tb, err) 134 | 135 | if !tb.IsBenchmark() { 136 | // More expensive result checks can be here. 137 | testutil.Equals(tb, int64(6221600000), ret) 138 | } 139 | } 140 | } 141 | 142 | // BenchmarkSum_fgprof recommended run options: 143 | // $ export ver=v1fg && go test -run '^$' -bench '^BenchmarkSum_fgprof' \ 144 | // -benchtime 60s -cpu 1 | tee ${ver}.txt 145 | // Read more in "Efficient Go"; Example 10-2. 146 | func BenchmarkSum_fgprof(b *testing.B) { 147 | f, err := os.Create("fgprof.pprof") 148 | testutil.Ok(b, err) 149 | 150 | defer func() { testutil.Ok(b, f.Close()) }() 151 | 152 | closeFn := fgprof.Start(f, fgprof.FormatPprof) 153 | BenchmarkSum(b) 154 | testutil.Ok(b, closeFn()) 155 | } 156 | 157 | // BenchmarkSum_AcrossInputs benchmarks the sum in "table test" fashion. 158 | // Read more in "Efficient Go"; Example 8-14. 159 | func BenchmarkSum_AcrossInputs(b *testing.B) { 160 | for _, tcase := range []struct { 161 | numLines int 162 | }{ 163 | {numLines: 0}, 164 | {numLines: 1e2}, 165 | {numLines: 1e4}, 166 | {numLines: 1e6}, 167 | {numLines: 2e6}, 168 | } { 169 | b.Run(fmt.Sprintf("lines-%d", tcase.numLines), func(b *testing.B) { 170 | b.ReportAllocs() 171 | 172 | fn := lazyCreateTestInput(b, tcase.numLines) 173 | 174 | b.ResetTimer() 175 | for i := 0; i < b.N; i++ { 176 | _, err := Sum(fn) 177 | testutil.Ok(b, err) 178 | } 179 | }) 180 | } 181 | } 182 | 183 | var sink string 184 | 185 | func BenchmarkStringConv(b *testing.B) { 186 | b.Run("string", func(b *testing.B) { 187 | b.ReportAllocs() 188 | 189 | s := make([]byte, 1e6) 190 | 191 | b.ResetTimer() 192 | for i := 0; i < b.N; i++ { 193 | sink = string(s) 194 | } 195 | }) 196 | 197 | b.Run("zero", func(b *testing.B) { 198 | b.ReportAllocs() 199 | 200 | s := make([]byte, 1e6) 201 | 202 | b.ResetTimer() 203 | for i := 0; i < b.N; i++ { 204 | sink = zeroCopyToString(s) 205 | } 206 | }) 207 | 208 | b.Run("zero?", func(b *testing.B) { 209 | b.ReportAllocs() 210 | 211 | s := make([]byte, 1e6) 212 | 213 | b.ResetTimer() 214 | for i := 0; i < b.N; i++ { 215 | sink = string(zeroCopyToString(s)) 216 | } 217 | }) 218 | 219 | b.Run("copy", func(b *testing.B) { 220 | b.ReportAllocs() 221 | 222 | s := make([]byte, 1e6) 223 | 224 | b.ResetTimer() 225 | for i := 0; i < b.N; i++ { 226 | c := make([]byte, 1e6) 227 | copy(c, s) 228 | sink = *((*string)(unsafe.Pointer(&c))) 229 | } 230 | }) 231 | } 232 | -------------------------------------------------------------------------------- /pkg/sum/labeler/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "encoding/json" 9 | "flag" 10 | stdlog "log" 11 | "net/http" 12 | "net/http/pprof" 13 | "os" 14 | "sync" 15 | "syscall" 16 | 17 | "github.com/efficientgo/core/errors" 18 | "github.com/efficientgo/examples/pkg/metrics/httpmidleware" 19 | "github.com/felixge/fgprof" 20 | "github.com/go-kit/log" 21 | "github.com/go-kit/log/level" 22 | "github.com/gobwas/pool/pbytes" 23 | "github.com/oklog/run" 24 | "github.com/prometheus/client_golang/prometheus" 25 | "github.com/prometheus/client_golang/prometheus/collectors" 26 | "github.com/prometheus/client_golang/prometheus/promhttp" 27 | "github.com/prometheus/common/version" 28 | "github.com/thanos-io/objstore/client" 29 | ) 30 | 31 | const ( 32 | labelObject1 = "labelObject1" 33 | labelObject2 = "labelObject2" 34 | labelObject3 = "labelObject3" 35 | labelObject4 = "labelObject4" 36 | ) 37 | 38 | var ( 39 | labelerFlags = flag.NewFlagSet("labeler-v1", flag.ExitOnError) 40 | addr = labelerFlags.String("listen-address", ":8080", "The address to listen on for HTTP requests.") 41 | objstoreConfigYAML = labelerFlags.String("objstore.config", "", "Configuration YAML for object storage to label objects against") 42 | labelerFunction = labelerFlags.String("function", "labelObjectNaive", "The function to use for labeling. labelObjectNaive, "+labelObject1+", "+labelObject2+", "+labelObject3+","+labelObject4) 43 | ) 44 | 45 | func main() { 46 | if err := runMain(context.Background(), os.Args[1:]); err != nil { 47 | // Use %+v for github.com/efficientgo/core/errors error to print with stack. 48 | stdlog.Fatalf("Error: %+v", errors.Wrapf(err, "%s", flag.Arg(0))) 49 | } 50 | } 51 | 52 | func runMain(ctx context.Context, args []string) (err error) { 53 | if err := labelerFlags.Parse(args); err != nil { 54 | return err 55 | } 56 | 57 | reg := prometheus.NewRegistry() 58 | reg.MustRegister( 59 | version.NewCollector("metrics"), 60 | collectors.NewGoCollector(), 61 | collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), 62 | ) 63 | if *objstoreConfigYAML == "" { 64 | return errors.New("missing -objstore.config flag") 65 | } 66 | 67 | logger := log.NewLogfmtLogger(os.Stderr) 68 | bkt, err := client.NewBucket(logger, []byte(*objstoreConfigYAML), reg, "labeler") 69 | if err != nil { 70 | return errors.Wrap(err, "bucket create") 71 | } 72 | 73 | l := &labeler{bkt: bkt} 74 | var labelObjectFunc labelFunc 75 | switch *labelerFunction { 76 | case "labelObjectNaive": 77 | l.tmpDir = "./tmp" 78 | if err := os.RemoveAll(l.tmpDir); err != nil { 79 | return errors.Wrap(err, "rm all") 80 | } 81 | if err := os.MkdirAll(l.tmpDir, os.ModePerm); err != nil { 82 | return errors.Wrap(err, "mkdir all") 83 | } 84 | 85 | labelObjectFunc = l.labelObjectNaive 86 | case labelObject1: 87 | labelObjectFunc = l.labelObject1 88 | case labelObject2: 89 | l.pool.New = func() any { return []byte(nil) } 90 | labelObjectFunc = l.labelObject2 91 | case labelObject3: 92 | l.bucketedPool = pbytes.New(1e3, 10e6) 93 | labelObjectFunc = l.labelObject3 94 | case labelObject4: 95 | // Yolo. 96 | labelerSet := [4]*labeler{ 97 | {bkt: bkt}, 98 | {bkt: bkt}, 99 | {bkt: bkt}, 100 | {bkt: bkt}, 101 | } 102 | var used [4]bool 103 | l := sync.Mutex{} 104 | 105 | labelObjectFunc = func(ctx context.Context, objID string) (label, error) { 106 | l.Lock() 107 | found := -1 108 | for i, u := range used { 109 | if u { 110 | continue 111 | } 112 | found = i 113 | } 114 | if found == -1 { 115 | l.Unlock() 116 | return label{}, errors.New("Did not expect more requests than 4 at the same time.") 117 | } 118 | used[found] = true 119 | l.Unlock() 120 | 121 | ret, err := labelerSet[found].labelObject4(ctx, objID) 122 | l.Lock() 123 | used[found] = false 124 | l.Unlock() 125 | return ret, err 126 | } 127 | default: 128 | return errors.Newf("unknown function %v", *labelerFunction) 129 | 130 | } 131 | 132 | metricMiddleware := httpmidleware.NewMiddleware(reg, nil) 133 | m := http.NewServeMux() 134 | m.Handle("/metrics", metricMiddleware.WrapHandler("/metric", promhttp.HandlerFor( 135 | reg, 136 | promhttp.HandlerOpts{ 137 | // Opt into OpenMetrics to support exemplars. 138 | EnableOpenMetrics: true, 139 | }, 140 | ))) 141 | m.HandleFunc("/label_object", metricMiddleware.WrapHandler("/label_object", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 142 | ctx := r.Context() 143 | w.Header().Add("Content-Type", "application/json; charset=utf-8") 144 | 145 | if err := r.ParseForm(); err != nil { 146 | httpErrHandle(w, http.StatusInternalServerError, err) 147 | } 148 | 149 | objectIDs := r.Form["object_id"] 150 | if len(objectIDs) == 0 { 151 | httpErrHandle(w, http.StatusBadRequest, errors.New("object_id parameter is required")) 152 | return 153 | } else if len(objectIDs) > 1 { 154 | httpErrHandle(w, http.StatusBadRequest, errors.New("only one object_id parameter is required")) 155 | return 156 | } 157 | 158 | // TODO(bwplotka): Discard request body. 159 | 160 | lbl, err := labelObjectFunc(ctx, objectIDs[0]) 161 | if err != nil { 162 | httpErrHandle(w, http.StatusInternalServerError, err) 163 | return 164 | } 165 | 166 | b, err := json.Marshal(&lbl) 167 | if err != nil { 168 | httpErrHandle(w, http.StatusInternalServerError, err) 169 | return 170 | } 171 | 172 | if _, err := w.Write(b); err != nil { 173 | httpErrHandle(w, http.StatusInternalServerError, err) 174 | return 175 | } 176 | }))) 177 | 178 | m.HandleFunc("/debug/pprof/", pprof.Index) 179 | m.HandleFunc("/debug/pprof/profile", pprof.Profile) 180 | m.HandleFunc("/debug/fgprof/profile", fgprof.Handler().ServeHTTP) 181 | 182 | srv := http.Server{Addr: *addr, Handler: m} 183 | 184 | g := &run.Group{} 185 | g.Add(func() error { 186 | level.Info(logger).Log("msg", "starting HTTP server", "addr", *addr) 187 | if err := srv.ListenAndServe(); err != nil { 188 | return errors.Wrap(err, "starting web server") 189 | } 190 | return nil 191 | }, func(error) { 192 | if err := srv.Close(); err != nil { 193 | level.Error(logger).Log("msg", "failed to stop web server", "err", err) 194 | } 195 | }) 196 | g.Add(run.SignalHandler(ctx, syscall.SIGINT, syscall.SIGTERM)) 197 | return g.Run() 198 | } 199 | 200 | func httpErrHandle(w http.ResponseWriter, code int, err error) { 201 | w.WriteHeader(code) 202 | _, _ = w.Write([]byte("{ \"error\": \" " + err.Error() + "\"}")) 203 | } 204 | 205 | type label struct { 206 | ObjID string `json:"object_id"` 207 | Sum int64 `json:"sum"` 208 | CheckSum []byte `json:"checksum"` 209 | } 210 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/efficientgo/examples 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/bwplotka/tracing-go v0.0.0-20230421061608-abdf862ceccd 7 | github.com/efficientgo/core v1.0.0-rc.2 8 | github.com/efficientgo/e2e v0.12.2-0.20220718133449-b567416bc99e 9 | github.com/felixge/fgprof v0.9.3 10 | github.com/go-kit/log v0.2.1 11 | github.com/gobwas/pool v0.2.1 12 | github.com/google/uuid v1.4.0 13 | github.com/oklog/run v1.1.0 14 | github.com/oklog/ulid v1.3.1 15 | github.com/prometheus/client_golang v1.17.0 16 | github.com/prometheus/common v0.45.0 17 | github.com/thanos-io/objstore v0.0.0-20220713125433-1d6b5f8ce8e8 18 | go.uber.org/goleak v1.3.0 19 | golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb 20 | golang.org/x/sys v0.15.0 21 | gopkg.in/yaml.v3 v3.0.1 22 | ) 23 | 24 | replace ( 25 | github.com/efficientgo/e2e => github.com/efficientgo/e2e v0.12.2-0.20220718133449-b567416bc99e 26 | // v0.x modules 27 | github.com/thanos-io/objstore => github.com/thanos-io/objstore v0.0.0-20220713125433-1d6b5f8ce8e8 28 | 29 | // OTLP did breaking change, pin it. 30 | go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.7.0 31 | go.opentelemetry.io/otel/exporters/jaeger => go.opentelemetry.io/otel/exporters/jaeger v1.6.3 32 | go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 33 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3 34 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace => go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.6.3 35 | go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.6.3 36 | ) 37 | 38 | require ( 39 | cloud.google.com/go v0.111.0 // indirect 40 | cloud.google.com/go/compute v1.23.3 // indirect 41 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 42 | cloud.google.com/go/iam v1.1.5 // indirect 43 | cloud.google.com/go/storage v1.35.1 // indirect 44 | github.com/Azure/azure-pipeline-go v0.2.3 // indirect 45 | github.com/Azure/azure-storage-blob-go v0.14.0 // indirect 46 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 47 | github.com/Azure/go-autorest/autorest v0.11.27 // indirect 48 | github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect 49 | github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect 50 | github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect 51 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect 52 | github.com/Azure/go-autorest/logger v0.2.1 // indirect 53 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect 54 | github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible // indirect 55 | github.com/aws/aws-sdk-go-v2 v1.24.0 // indirect 56 | github.com/aws/aws-sdk-go-v2/config v1.26.1 // indirect 57 | github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect 58 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect 59 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect 60 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect 61 | github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect 62 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect 63 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect 64 | github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect 65 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect 66 | github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect 67 | github.com/aws/smithy-go v1.19.0 // indirect 68 | github.com/baidubce/bce-sdk-go v0.9.160 // indirect 69 | github.com/beorn7/perks v1.0.1 // indirect 70 | github.com/cenkalti/backoff/v4 v4.1.2 // indirect 71 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 72 | github.com/clbanning/mxj v1.8.4 // indirect 73 | github.com/davecgh/go-spew v1.1.1 // indirect 74 | github.com/dimchansky/utfbom v1.1.1 // indirect 75 | github.com/dustin/go-humanize v1.0.1 // indirect 76 | github.com/efficientgo/tools/core v0.0.0-20230505153745-6b7392939a60 // indirect 77 | github.com/go-logfmt/logfmt v0.6.0 // indirect 78 | github.com/go-logr/logr v1.3.0 // indirect 79 | github.com/go-logr/stdr v1.2.2 // indirect 80 | github.com/golang-jwt/jwt/v4 v4.5.0 // indirect 81 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 82 | github.com/golang/protobuf v1.5.3 // indirect 83 | github.com/google/go-cmp v0.6.0 // indirect 84 | github.com/google/go-querystring v1.1.0 // indirect 85 | github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 // indirect 86 | github.com/google/s2a-go v0.1.7 // indirect 87 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 88 | github.com/googleapis/gax-go/v2 v2.12.0 // indirect 89 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect 90 | github.com/jpillora/backoff v1.0.0 // indirect 91 | github.com/json-iterator/go v1.1.12 // indirect 92 | github.com/klauspost/compress v1.17.4 // indirect 93 | github.com/klauspost/cpuid/v2 v2.2.6 // indirect 94 | github.com/mattn/go-ieproxy v0.0.1 // indirect 95 | github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect 96 | github.com/minio/md5-simd v1.1.2 // indirect 97 | github.com/minio/minio-go/v7 v7.0.65 // indirect 98 | github.com/minio/sha256-simd v1.0.1 // indirect 99 | github.com/mitchellh/go-homedir v1.1.0 // indirect 100 | github.com/mitchellh/mapstructure v1.5.0 // indirect 101 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 102 | github.com/modern-go/reflect2 v1.0.2 // indirect 103 | github.com/mozillazg/go-httpheader v0.4.0 // indirect 104 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect 105 | github.com/ncw/swift v1.0.53 // indirect 106 | github.com/opentracing/opentracing-go v1.2.0 // indirect 107 | github.com/pkg/errors v0.9.1 // indirect 108 | github.com/pmezard/go-difflib v1.0.0 // indirect 109 | github.com/prometheus/client_model v0.5.0 // indirect 110 | github.com/prometheus/procfs v0.12.0 // indirect 111 | github.com/rs/xid v1.5.0 // indirect 112 | github.com/sirupsen/logrus v1.9.3 // indirect 113 | github.com/tencentyun/cos-go-sdk-v5 v0.7.45 // indirect 114 | go.opencensus.io v0.24.0 // indirect 115 | go.opentelemetry.io/otel v1.19.0 // indirect 116 | go.opentelemetry.io/otel/exporters/jaeger v1.6.3 // indirect 117 | go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3 // indirect 118 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 // indirect 119 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3 // indirect 120 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 // indirect 121 | go.opentelemetry.io/otel/sdk v1.19.0 // indirect 122 | go.opentelemetry.io/otel/trace v1.19.0 // indirect 123 | go.opentelemetry.io/proto/otlp v0.15.0 // indirect 124 | golang.org/x/crypto v0.16.0 // indirect 125 | golang.org/x/net v0.19.0 // indirect 126 | golang.org/x/oauth2 v0.15.0 // indirect 127 | golang.org/x/sync v0.5.0 // indirect 128 | golang.org/x/text v0.14.0 // indirect 129 | golang.org/x/time v0.5.0 // indirect 130 | google.golang.org/api v0.153.0 // indirect 131 | google.golang.org/appengine v1.6.8 // indirect 132 | google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 // indirect 133 | google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect 134 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect 135 | google.golang.org/grpc v1.59.0 // indirect 136 | google.golang.org/protobuf v1.31.0 // indirect 137 | gopkg.in/ini.v1 v1.67.0 // indirect 138 | gopkg.in/yaml.v2 v2.4.0 // indirect 139 | ) 140 | -------------------------------------------------------------------------------- /pkg/sum/sum.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package sum 5 | 6 | import ( 7 | "bufio" 8 | "bytes" 9 | "io" 10 | "os" 11 | "strconv" 12 | "unsafe" 13 | 14 | "github.com/efficientgo/core/errcapture" 15 | "github.com/efficientgo/core/errors" 16 | ) 17 | 18 | // Sum is a naive implementation and algorithm for summing integers from file. 19 | // Read more in "Efficient Go"; Example 4-1. 20 | func Sum(fileName string) (ret int64, _ error) { 21 | b, err := os.ReadFile(fileName) 22 | if err != nil { 23 | return 0, err 24 | } 25 | for _, line := range bytes.Split(b, []byte("\n")) { 26 | if len(line) == 0 { 27 | // Empty line at the end. 28 | continue 29 | } 30 | 31 | num, err := strconv.ParseInt(string(line), 10, 64) 32 | if err != nil { 33 | return 0, err 34 | } 35 | 36 | ret += num 37 | } 38 | return ret, nil 39 | } 40 | 41 | // Sum2 is sum with optimized the first latency + CPU bottleneck bytes.Split. 42 | // bytes.Split look complex to hande different cases. It allocates a lot causing It looks like the algo is simple enough to just 43 | // implement on our own (tried scanner := bufio.NewScanner(f) but it's slower). 44 | // 30% less latency and 5x less memory than Sum. 45 | // Read more in "Efficient Go"; Example 10-3. 46 | func Sum2(fileName string) (ret int64, _ error) { 47 | b, err := os.ReadFile(fileName) 48 | if err != nil { 49 | return 0, err 50 | } 51 | 52 | var last int 53 | for i := 0; i < len(b); i++ { 54 | if b[i] != '\n' { 55 | continue 56 | } 57 | num, err := strconv.ParseInt(string(b[last:i]), 10, 64) 58 | if err != nil { 59 | return 0, err 60 | } 61 | 62 | ret += num 63 | last = i + 1 64 | } 65 | return ret, nil 66 | } 67 | 68 | // Sum2_scanner is a sum attempting using scanner. Actually slower than Sum2, but uses less memory. 69 | func Sum2_scanner(fileName string) (ret int64, err error) { 70 | f, err := os.Open(fileName) 71 | if err != nil { 72 | return 0, err 73 | } 74 | defer errcapture.Do(&err, f.Close, "close file") 75 | 76 | scanner := bufio.NewScanner(f) 77 | for scanner.Scan() { 78 | num, err := strconv.ParseInt(string(scanner.Bytes()), 10, 64) 79 | if err != nil { 80 | return 0, err 81 | } 82 | 83 | ret += num 84 | } 85 | return ret, nil 86 | } 87 | 88 | func zeroCopyToString(b []byte) string { 89 | return *((*string)(unsafe.Pointer(&b))) 90 | } 91 | 92 | // Sum3 is a sum with optimized the second latency + CPU bottleneck: string conversion. 93 | // On CPU profile we see byte to string conversion not only allocate memory, but also takes precious time. 94 | // Let's perform zeroCopy conversion. 95 | // 2x less latency memory than Sum2. 96 | // Read more in "Efficient Go"; Example 10-4. 97 | func Sum3(fileName string) (ret int64, _ error) { 98 | b, err := os.ReadFile(fileName) 99 | if err != nil { 100 | return 0, err 101 | } 102 | 103 | var last int 104 | for i := 0; i < len(b); i++ { 105 | if b[i] != '\n' { 106 | continue 107 | } 108 | num, err := strconv.ParseInt(zeroCopyToString(b[last:i]), 10, 64) 109 | if err != nil { 110 | return 0, err 111 | } 112 | 113 | ret += num 114 | last = i + 1 115 | } 116 | return ret, nil 117 | } 118 | 119 | // ParseInt is 3-4x times faster than strconv.ParseInt or Atoi. 120 | func ParseInt(input []byte) (n int64, _ error) { 121 | factor := int64(1) 122 | k := 0 123 | 124 | // TODO(bwplotka): Optimize if only positive integers are accepted (only 2.6% overhead in my tests though). 125 | if input[0] == '-' { 126 | factor *= -1 127 | k++ 128 | } 129 | 130 | for i := len(input) - 1; i >= k; i-- { 131 | if input[i] < '0' || input[i] > '9' { 132 | return 0, errors.Newf("not a valid integer: %v", input) 133 | } 134 | 135 | n += factor * int64(input[i]-'0') 136 | factor *= 10 137 | } 138 | return n, nil 139 | } 140 | 141 | // Sum4 is a sum with optimized the second latency + CPU bottleneck: ParseInt and string conversion. 142 | // On CPU profile we see that ParseInt does a lot of checks that we might not need. We write our own parsing 143 | // straight from byte to avoid conversion CPU time. 144 | // 2x less latency, same mem as Sum3. 145 | // Read more in "Efficient Go"; Example 10-5. 146 | func Sum4(fileName string) (ret int64, err error) { 147 | b, err := os.ReadFile(fileName) 148 | if err != nil { 149 | return 0, err 150 | } 151 | 152 | var last int 153 | for i := 0; i < len(b); i++ { 154 | if b[i] != '\n' { 155 | continue 156 | } 157 | num, err := ParseInt(b[last:i]) 158 | if err != nil { 159 | return 0, err 160 | } 161 | 162 | ret += num 163 | last = i + 1 164 | } 165 | return ret, nil 166 | } 167 | 168 | func Sum4_atoi(fileName string) (ret int64, err error) { 169 | b, err := os.ReadFile(fileName) 170 | if err != nil { 171 | return 0, err 172 | } 173 | 174 | var last int 175 | for i := 0; i < len(b); i++ { 176 | if b[i] != '\n' { 177 | continue 178 | } 179 | num, err := strconv.Atoi(zeroCopyToString(b[last:i])) 180 | if err != nil { 181 | return 0, err 182 | } 183 | 184 | ret += int64(num) 185 | last = i + 1 186 | } 187 | return ret, nil 188 | } 189 | 190 | // Sum5 is like Sum4, but noticing that it takes time to even allocate 21 MB on heap (and read file to it). 191 | // Let's try to use scanner instead. 192 | // Slower than Sum4 and Sum6 because scanner is not optimized for this...? Scanner takes 73% of CPU time. 193 | // Read more in "Efficient Go"; Example 10-7. 194 | func Sum5(fileName string) (ret int64, err error) { 195 | f, err := os.Open(fileName) 196 | if err != nil { 197 | return 0, err 198 | } 199 | defer errcapture.Do(&err, f.Close, "close file") 200 | 201 | scanner := bufio.NewScanner(f) 202 | for scanner.Scan() { 203 | num, err := ParseInt(scanner.Bytes()) 204 | if err != nil { 205 | return 0, err 206 | } 207 | 208 | ret += num 209 | } 210 | return ret, scanner.Err() 211 | } 212 | 213 | func Sum5_line(fileName string) (ret int64, err error) { 214 | f, err := os.Open(fileName) 215 | if err != nil { 216 | return 0, err 217 | } 218 | defer errcapture.Do(&err, f.Close, "close file") 219 | 220 | scanner := bufio.NewScanner(f) 221 | scanner.Split(ScanLines) 222 | for scanner.Scan() { 223 | num, err := ParseInt(scanner.Bytes()) 224 | if err != nil { 225 | return 0, err 226 | } 227 | 228 | ret += num 229 | } 230 | return ret, scanner.Err() 231 | } 232 | 233 | func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { 234 | if atEOF && len(data) == 0 { 235 | return 0, nil, nil 236 | } 237 | for i := range data { 238 | if data[i] != '\n' { 239 | continue 240 | } 241 | return i + 1, data[0:i], nil 242 | } 243 | // If we're at EOF, we have a final, non-terminated line. Return it. 244 | if atEOF { 245 | return len(data), data, nil 246 | } 247 | 248 | // Request more data. 249 | return 0, nil, nil 250 | } 251 | 252 | // Sum6 is like Sum4, but trying to use max 10 KB of mem without scanner and bulk read. 253 | // Assuming no integer is larger than 8 000 digits. 254 | // Read more in "Efficient Go"; Example 10-8. 255 | func Sum6(fileName string) (ret int64, err error) { 256 | f, err := os.Open(fileName) 257 | if err != nil { 258 | return 0, err 259 | } 260 | defer errcapture.Do(&err, f.Close, "close file") 261 | 262 | buf := make([]byte, 8*1024) 263 | return Sum6Reader(f, buf) 264 | } 265 | 266 | func Sum6Reader(r io.Reader, buf []byte) (ret int64, err error) { // Just inlining this function saves 7% on latency 267 | var offset, n int 268 | for err != io.EOF { 269 | n, err = r.Read(buf[offset:]) 270 | if err != nil && err != io.EOF { 271 | return 0, err 272 | } 273 | n += offset 274 | 275 | var last int 276 | //for i := 0; i < n; i++ { // Funny enough this is 5% slower! 277 | for i := range buf[:n] { 278 | if buf[i] != '\n' { 279 | continue 280 | } 281 | num, err := ParseInt(buf[last:i]) 282 | if err != nil { 283 | return 0, err 284 | } 285 | 286 | ret += num 287 | last = i + 1 288 | } 289 | 290 | offset = n - last 291 | if offset > 0 { 292 | _ = copy(buf, buf[last:n]) 293 | } 294 | } 295 | return ret, nil 296 | } 297 | 298 | var sumByFile = map[string]int64{} 299 | 300 | // Sum7 is cached (cheating!) (: 301 | // Read more in "Efficient Go"; Example 10-15. 302 | func Sum7(fileName string) (int64, error) { 303 | if s, ok := sumByFile[fileName]; ok { 304 | return s, nil 305 | } 306 | 307 | ret, err := Sum(fileName) 308 | if err != nil { 309 | return 0, err 310 | } 311 | 312 | sumByFile[fileName] = ret 313 | return ret, nil 314 | } 315 | -------------------------------------------------------------------------------- /pkg/generics/blocks_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package generics 5 | 6 | import ( 7 | "sort" 8 | "testing" 9 | "time" 10 | 11 | "github.com/efficientgo/core/testutil" 12 | ) 13 | 14 | type sortable []Block 15 | 16 | func (s sortable) Len() int { return len(s) } 17 | func (s sortable) Less(i, j int) bool { return s[i].Compare(s[j]) > 0 } 18 | func (s sortable) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 19 | 20 | func Example2() { 21 | toSort := []Block{ /* ... */ } 22 | sort.Sort(sortable(toSort)) 23 | 24 | toSort2 := []Block{ /* ... */ } 25 | genericSort[Block](toSort2) 26 | 27 | // Output: 28 | } 29 | 30 | func TestSortBlocks(t *testing.T) { 31 | n := time.Now() 32 | expected := []Block{ 33 | {start: n.Add(-10 * time.Hour)}, 34 | {start: n}, 35 | {start: n.Add(5 * time.Hour)}, 36 | {start: n.Add(8 * time.Hour)}, 37 | {start: n.Add(20 * time.Hour)}, 38 | } 39 | unsorted := []Block{ 40 | {start: n.Add(20 * time.Hour)}, 41 | {start: n.Add(5 * time.Hour)}, 42 | {start: n}, 43 | {start: n.Add(8 * time.Hour)}, 44 | {start: n.Add(-10 * time.Hour)}, 45 | } 46 | 47 | t.Run("sortable", func(t *testing.T) { 48 | b := make([]Block, len(unsorted)) 49 | copy(b, unsorted) 50 | 51 | sort.Sort(sortable(b)) 52 | 53 | testutil.Equals(t, expected, b) 54 | }) 55 | t.Run("sort.Slice", func(t *testing.T) { 56 | b := make([]Block, len(unsorted)) 57 | copy(b, unsorted) 58 | 59 | sort.Slice(b, func(i, j int) bool { 60 | return b[i].start.Before(b[j].start) 61 | }) 62 | 63 | testutil.Equals(t, expected, b) 64 | }) 65 | t.Run("generics", func(t *testing.T) { 66 | b := make([]Block, len(unsorted)) 67 | copy(b, unsorted) 68 | 69 | genericSort[Block](b) 70 | 71 | testutil.Equals(t, expected, b) 72 | }) 73 | } 74 | 75 | func BenchmarkSortBlock(b *testing.B) { 76 | n := time.Now() 77 | unsorted := make([]Block, 1e6) 78 | for i := range unsorted { 79 | unsorted[i].start = n.Add(-1 * time.Duration(i)) 80 | } 81 | 82 | b.Run("sortable", func(b *testing.B) { 83 | b.ReportAllocs() 84 | b.ResetTimer() 85 | for i := 0; i < b.N; i++ { 86 | b.StopTimer() 87 | toSort := make([]Block, len(unsorted)) 88 | copy(toSort, unsorted) 89 | b.StartTimer() 90 | 91 | sort.Sort(sortable(toSort)) 92 | } 93 | }) 94 | b.Run("sort.Slice", func(b *testing.B) { 95 | b.ReportAllocs() 96 | b.ResetTimer() 97 | for i := 0; i < b.N; i++ { 98 | b.StopTimer() 99 | toSort := make([]Block, len(unsorted)) 100 | copy(toSort, unsorted) 101 | b.StartTimer() 102 | 103 | sort.Slice(toSort, func(i, j int) bool { 104 | return toSort[i].start.Before(toSort[j].start) 105 | }) 106 | } 107 | }) 108 | b.Run("generics", func(b *testing.B) { 109 | b.ReportAllocs() 110 | b.ResetTimer() 111 | for i := 0; i < b.N; i++ { 112 | b.StopTimer() 113 | toSort := make([]Block, len(unsorted)) 114 | copy(toSort, unsorted) 115 | b.StartTimer() 116 | 117 | genericSort[Block](toSort) 118 | } 119 | }) 120 | } 121 | 122 | func insertionSortInterface(data sort.Interface) { 123 | for i := 1; i < data.Len(); i++ { 124 | for j := i; j > 0 && data.Less(j, j-1); j-- { 125 | data.Swap(j, j-1) 126 | } 127 | } 128 | } 129 | 130 | func insertionSortGeneric[T Comparable[T]](data []T) { 131 | for i := 1; i < len(data); i++ { 132 | for j := i; j > 0 && data[j].Compare(data[j-1]) > 0; j-- { 133 | data[j], data[j-1] = data[j-1], data[j] 134 | } 135 | } 136 | } 137 | 138 | func insertionSortBlocks(data []Block) { 139 | for i := 1; i < len(data); i++ { 140 | for j := i; j > 0 && data[j].Compare(data[j-1]) > 0; j-- { 141 | data[j], data[j-1] = data[j-1], data[j] 142 | } 143 | } 144 | } 145 | 146 | type compareFunc[T any] func(a, b T) int 147 | 148 | func insertionSortGeneric2[T any](data []T, cmpFunc compareFunc[T]) { 149 | for i := 1; i < len(data); i++ { 150 | for j := i; j > 0 && cmpFunc(data[j], data[j-1]) > 0; j-- { 151 | data[j], data[j-1] = data[j-1], data[j] 152 | } 153 | } 154 | } 155 | 156 | func CompareBlocks(a, b Block) int { 157 | if a.start.Before(b.start) { 158 | return 1 159 | } 160 | if a.start.Equal(b.start) { 161 | return 0 162 | } 163 | return -1 164 | } 165 | 166 | type lessFunc[T any] func(a, b T) bool 167 | 168 | func insertionSortGenericLess[T any](data []T, lessFunc lessFunc[T]) { 169 | for i := 1; i < len(data); i++ { 170 | for j := i; j > 0 && lessFunc(data[j], data[j-1]); j-- { 171 | data[j], data[j-1] = data[j-1], data[j] 172 | } 173 | } 174 | } 175 | 176 | func LessBlocks(a, b Block) bool { 177 | return a.start.Before(b.start) 178 | } 179 | 180 | func insertionSortBlocksLess(data []Block, lessFunc func(a, b Block) bool) { 181 | for i := 1; i < len(data); i++ { 182 | for j := i; j > 0 && lessFunc(data[j], data[j-1]); j-- { 183 | data[j], data[j-1] = data[j-1], data[j] 184 | } 185 | } 186 | } 187 | 188 | func TestInsertionSortBlocks(t *testing.T) { 189 | n := time.Now() 190 | expected := []Block{ 191 | {start: n.Add(-10 * time.Hour)}, 192 | {start: n}, 193 | {start: n.Add(5 * time.Hour)}, 194 | {start: n.Add(8 * time.Hour)}, 195 | {start: n.Add(20 * time.Hour)}, 196 | } 197 | unsorted := []Block{ 198 | {start: n.Add(20 * time.Hour)}, 199 | {start: n.Add(5 * time.Hour)}, 200 | {start: n}, 201 | {start: n.Add(8 * time.Hour)}, 202 | {start: n.Add(-10 * time.Hour)}, 203 | } 204 | 205 | t.Run("sortable", func(t *testing.T) { 206 | b := make([]Block, len(unsorted)) 207 | copy(b, unsorted) 208 | 209 | insertionSortInterface(sortable(b)) 210 | 211 | testutil.Equals(t, expected, b) 212 | }) 213 | t.Run("generics", func(t *testing.T) { 214 | b := make([]Block, len(unsorted)) 215 | copy(b, unsorted) 216 | 217 | insertionSortGeneric[Block](b) 218 | 219 | testutil.Equals(t, expected, b) 220 | }) 221 | t.Run("blocks", func(t *testing.T) { 222 | b := make([]Block, len(unsorted)) 223 | copy(b, unsorted) 224 | 225 | insertionSortBlocks(b) 226 | 227 | testutil.Equals(t, expected, b) 228 | }) 229 | t.Run("generics_func", func(t *testing.T) { 230 | b := make([]Block, len(unsorted)) 231 | copy(b, unsorted) 232 | 233 | insertionSortGeneric2[Block](b, CompareBlocks) 234 | 235 | testutil.Equals(t, expected, b) 236 | }) 237 | t.Run("generics_less", func(t *testing.T) { 238 | b := make([]Block, len(unsorted)) 239 | copy(b, unsorted) 240 | 241 | insertionSortGenericLess[Block](b, LessBlocks) 242 | 243 | testutil.Equals(t, expected, b) 244 | }) 245 | t.Run("blocks_less", func(t *testing.T) { 246 | b := make([]Block, len(unsorted)) 247 | copy(b, unsorted) 248 | 249 | insertionSortBlocksLess(b, LessBlocks) 250 | 251 | testutil.Equals(t, expected, b) 252 | }) 253 | } 254 | 255 | func BenchmarkInsertionSortBlock(b *testing.B) { 256 | n := time.Now() 257 | unsorted := make([]Block, 1e4) 258 | for i := range unsorted { 259 | unsorted[i].start = n.Add(-1 * time.Duration(i)) 260 | } 261 | 262 | b.Run("sortable", func(b *testing.B) { 263 | b.ResetTimer() 264 | for i := 0; i < b.N; i++ { 265 | b.StopTimer() 266 | toSort := make([]Block, len(unsorted)) 267 | copy(toSort, unsorted) 268 | b.StartTimer() 269 | 270 | insertionSortInterface(sortable(toSort)) 271 | } 272 | }) 273 | b.Run("generics", func(b *testing.B) { 274 | b.ResetTimer() 275 | for i := 0; i < b.N; i++ { 276 | b.StopTimer() 277 | toSort := make([]Block, len(unsorted)) 278 | copy(toSort, unsorted) 279 | b.StartTimer() 280 | 281 | insertionSortGeneric[Block](toSort) 282 | } 283 | }) 284 | b.Run("blocks", func(b *testing.B) { 285 | b.ResetTimer() 286 | for i := 0; i < b.N; i++ { 287 | b.StopTimer() 288 | toSort := make([]Block, len(unsorted)) 289 | copy(toSort, unsorted) 290 | b.StartTimer() 291 | 292 | insertionSortBlocks(toSort) 293 | } 294 | }) 295 | b.Run("generics_func", func(b *testing.B) { 296 | b.ResetTimer() 297 | for i := 0; i < b.N; i++ { 298 | b.StopTimer() 299 | toSort := make([]Block, len(unsorted)) 300 | copy(toSort, unsorted) 301 | b.StartTimer() 302 | 303 | insertionSortGeneric2[Block](toSort, CompareBlocks) 304 | } 305 | }) 306 | b.Run("generics_less", func(b *testing.B) { 307 | b.ResetTimer() 308 | for i := 0; i < b.N; i++ { 309 | b.StopTimer() 310 | toSort := make([]Block, len(unsorted)) 311 | copy(toSort, unsorted) 312 | b.StartTimer() 313 | 314 | insertionSortGenericLess[Block](toSort, LessBlocks) 315 | } 316 | }) 317 | b.Run("blocks_less", func(b *testing.B) { 318 | b.ResetTimer() 319 | for i := 0; i < b.N; i++ { 320 | b.StopTimer() 321 | toSort := make([]Block, len(unsorted)) 322 | copy(toSort, unsorted) 323 | b.StartTimer() 324 | 325 | insertionSortBlocksLess(toSort, LessBlocks) 326 | } 327 | }) 328 | } 329 | -------------------------------------------------------------------------------- /pkg/metrics/latency_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package metrics 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | stdlog "log" 10 | "net/http" 11 | "os" 12 | "sync" 13 | "testing" 14 | "time" 15 | 16 | "github.com/bwplotka/tracing-go/tracing" 17 | "github.com/bwplotka/tracing-go/tracing/exporters/jaeger" 18 | "github.com/bwplotka/tracing-go/tracing/exporters/otlp" 19 | "github.com/efficientgo/core/testutil" 20 | "github.com/efficientgo/e2e" 21 | e2einteractive "github.com/efficientgo/e2e/interactive" 22 | e2emonitoring "github.com/efficientgo/e2e/monitoring" 23 | "github.com/go-kit/log" 24 | "github.com/go-kit/log/level" 25 | "github.com/prometheus/client_golang/prometheus" 26 | "github.com/prometheus/client_golang/prometheus/promauto" 27 | "github.com/prometheus/client_golang/prometheus/promhttp" 28 | ) 29 | 30 | const xTimes = 10 31 | 32 | // ExampleLatencySimplest is the simplest way of getting information about latency of your operation. 33 | // Read more in "Efficient Go"; Example 6-1. 34 | func ExampleLatencySimplest() { 35 | prepare() 36 | 37 | for i := 0; i < xTimes; i++ { 38 | start := time.Now() 39 | err := doOperation() // Operation we want to measure and potentially optimize... 40 | elapsed := time.Since(start) 41 | 42 | fmt.Printf("%v ns\n", elapsed.Nanoseconds()) 43 | 44 | if err != nil { /* Handle error... */ 45 | } 46 | } 47 | 48 | tearDown() 49 | } 50 | 51 | // ExampleLatencyAggregated shows a way of getting information about aggregated latency of your operation. 52 | // Read more in "Efficient Go"; Example 6-2. 53 | func ExampleLatencyAggregated() { 54 | var count, sum int64 55 | 56 | prepare() 57 | 58 | for i := 0; i < xTimes; i++ { 59 | start := time.Now() 60 | err := doOperation() // Operation we want to measure and potentially optimize... 61 | elapsed := time.Since(start) 62 | 63 | sum += elapsed.Nanoseconds() 64 | count++ 65 | 66 | if err != nil { /* Handle error... */ 67 | } 68 | } 69 | 70 | fmt.Printf("%v ns/op\n", sum/count) // 188324467 ns/op 71 | 72 | tearDown() 73 | } 74 | 75 | // Simplest benchmark. 76 | // Read more in "Efficient Go"; Example 6-3. 77 | func BenchmarkExampleLatency(b *testing.B) { 78 | prepare() 79 | 80 | b.ResetTimer() 81 | for i := 0; i < b.N; i++ { 82 | _ = doOperation() 83 | } 84 | } 85 | 86 | // Example of getting operation latency and exposing it through a log line metric. 87 | // Read more in "Efficient Go"; Example 6-4. 88 | func ExampleLatencyLog() { 89 | logger := log.With(log.NewLogfmtLogger(os.Stderr), "ts", log.DefaultTimestampUTC) 90 | 91 | prepare() 92 | 93 | for i := 0; i < xTimes; i++ { 94 | now := time.Now() 95 | err := doOperation() // Operation we want to measure and potentially optimize... 96 | elapsed := time.Since(now) 97 | 98 | // Log line level=info ts=2022-05-02T11:30:47.803680146Z msg="finished operation" result="error first" elapsed=292.639849ms 99 | level.Info(logger).Log("msg", "finished operation", "result", err, "elapsed", elapsed.String()) 100 | 101 | if err != nil { /* Handle error... */ 102 | } 103 | } 104 | 105 | tearDown() 106 | } 107 | 108 | // Example of getting operation latency from tracing instrumentation. 109 | // Read more in "Efficient Go"; Example 6-6. 110 | func ExampleLatencyTrace() { 111 | tracer, cleanFn, err := tracing.NewTracer(otlp.Exporter("")) 112 | if err != nil { /* Handle error... */ 113 | } 114 | defer cleanFn() 115 | 116 | prepare() 117 | 118 | for i := 0; i < xTimes; i++ { 119 | ctx, span := tracer.StartSpan("doOperation") 120 | err := doOperationWithCtx(ctx) // Operation we want to measure and potentially optimize... 121 | span.End(err) 122 | 123 | if err != nil { /* Handle error... */ 124 | } 125 | } 126 | 127 | tearDown() 128 | } 129 | 130 | // Example of getting operation latency and exposing it through Prometheus metric. 131 | // Read more in "Efficient Go"; Example 6-7. 132 | func ExampleLatencyMetric() { 133 | reg := prometheus.NewRegistry() 134 | latencySeconds := promauto.With(reg). 135 | NewHistogramVec(prometheus.HistogramOpts{ 136 | Name: "operation_duration_seconds", 137 | Help: "Tracks the latency of operations in seconds.", 138 | Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100}, 139 | }, []string{"error_type"}) 140 | 141 | prepare() 142 | 143 | go func() { 144 | for i := 0; i < xTimes; i++ { 145 | now := time.Now() 146 | err := doOperation() // Operation we want to measure and potentially optimize... 147 | elapsed := time.Since(now) 148 | 149 | // Prometheus metric. 150 | latencySeconds.WithLabelValues(errorType(err)). 151 | Observe(elapsed.Seconds()) 152 | 153 | if err != nil { /* Handle error... */ 154 | } 155 | 156 | time.Sleep(1 * time.Second) 157 | } 158 | }() 159 | 160 | if err := http.ListenAndServe( 161 | ":8080", 162 | promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), 163 | ); err != nil { 164 | stdlog.Fatal(err) 165 | } 166 | 167 | tearDown() 168 | 169 | printPrometheusMetrics(reg) 170 | } 171 | 172 | func TestLatencyE2e(t *testing.T) { 173 | t.Skip("Comment this line if you want to run it - it's interactive test. Won't be useful in CI") 174 | 175 | e, err := e2e.NewDockerEnvironment("e2e_latency") 176 | testutil.Ok(t, err) 177 | t.Cleanup(e.Close) 178 | 179 | reg := prometheus.NewRegistry() 180 | mon, err := e2emonitoring.Start(e, 181 | e2emonitoring.WithScrapeInterval(1*time.Second), 182 | e2emonitoring.WithCustomRegistry(reg), 183 | ) 184 | testutil.Ok(t, err) 185 | 186 | // Setup in-memory Jaeger to check if backend can understand our client traces. 187 | j := e.Runnable("tracing"). 188 | WithPorts( 189 | map[string]int{ 190 | "http.front": 16686, 191 | "jaeger.thrift": 16000, 192 | }). 193 | Init(e2e.StartOptions{ 194 | Image: "jaegertracing/all-in-one:1.33", 195 | Command: e2e.NewCommand("--collector.http-server.host-port=:16000"), 196 | }) 197 | 198 | testutil.Ok(t, e2e.StartAndWaitReady(j)) 199 | 200 | tracer, cleanFn, err := tracing.NewTracer( 201 | jaeger.Exporter("http://"+j.Endpoint("jaeger.thrift")+"/api/traces"), 202 | tracing.WithServiceName("example"), 203 | ) 204 | testutil.Ok(t, err) 205 | t.Cleanup(func() { cleanFn() }) 206 | 207 | logger := log.With(log.NewLogfmtLogger(os.Stderr), "ts", log.DefaultTimestampUTC) 208 | latencySeconds := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ 209 | Name: "operation_duration_seconds", 210 | Help: "Tracks the latency of operations in seconds.", 211 | Buckets: []float64{0.001, 0.01, 0.1, 1, 10, 100}, 212 | }, []string{"error_type"}) 213 | 214 | prepare() 215 | 216 | var wg sync.WaitGroup 217 | wg.Add(1) 218 | ctx, cancel := context.WithCancel(context.Background()) 219 | go func() { 220 | for ctx.Err() == nil { 221 | time.Sleep(50 * time.Millisecond) 222 | 223 | // Instrumentation COMBO! 224 | sctx, span := tracer.StartSpan("doOperation", tracing.WithTracerStartSpanContext(ctx)) 225 | start := time.Now() 226 | err := doOperationWithCtx(sctx) // Operation we want to measure and potentially optimize... 227 | elapsed := time.Since(start) 228 | span.End(err) 229 | 230 | level.Info(logger).Log("msg", "finished operation", "result", err, "elapsed", elapsed.String()) 231 | 232 | if span.Context().IsSampled() { 233 | latencySeconds.WithLabelValues(errorType(err)).(prometheus.ExemplarObserver).ObserveWithExemplar( 234 | elapsed.Seconds(), map[string]string{"traceID": span.Context().TraceID()}) 235 | } else { 236 | latencySeconds.WithLabelValues(errorType(err)).Observe(elapsed.Seconds()) 237 | } 238 | 239 | // Handle error... 240 | if err != nil { 241 | } 242 | } 243 | wg.Done() 244 | }() 245 | 246 | // TODO(bwplotka): Make it non-interactive and expect certain Jaeger output. 247 | testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+j.Endpoint("http.front"))) 248 | testutil.Ok(t, mon.OpenUserInterfaceInBrowser("/graph?g0.expr=rate(operation_duration_seconds_sum%5B1m%5D)%20%2F%20rate(operation_duration_seconds_count%5B1m%5D)&g0.tab=0&g0.stacked=0&g0.range_input=10m&g0.end_input=2022-04-09%2020%3A20%3A40&g0.moment_input=2022-04-09%2020%3A20%3A40&g1.expr=histogram_quantile(0.9%2C%20sum%20by(error_type%2C%20le)%20(rate(operation_duration_seconds_bucket%5B1m%5D)))&g1.tab=0&g1.stacked=0&g1.range_input=10m&g1.end_input=2022-04-09%2020%3A20%3A40&g1.moment_input=2022-04-09%2020%3A20%3A40&g2.expr=operation_duration_seconds_bucket&g2.tab=1&g2.stacked=0&g2.range_input=1h&g3.expr=delta(operation_duration_seconds_count%5B1m%5D)&g3.tab=0&g3.stacked=0&g3.range_input=15m")) 249 | testutil.Ok(t, e2einteractive.RunUntilEndpointHit()) 250 | 251 | cancel() 252 | wg.Wait() 253 | 254 | tearDown() 255 | } 256 | 257 | var latTest time.Duration 258 | 259 | func BenchmarkLatencyItself(b *testing.B) { 260 | for i := 0; i < b.N; i++ { 261 | start := time.Now() 262 | latTest = time.Since(start) 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /pkg/memory/mmap/interactive/interactive_open.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "os" 10 | "runtime" 11 | ) 12 | 13 | /* 14 | command-line-arguments 15 | ./interactive2.go:11:30: inlining call to os.Getpid 16 | ./interactive2.go:11:30: inlining call to syscall.Getpid 17 | ./interactive2.go:11:13: inlining call to fmt.Println 18 | ./interactive2.go:16:19: inlining call to os.Open 19 | ./interactive2.go:61:13: inlining call to fmt.Println 20 | ./interactive2.go:62:12: inlining call to fmt.Scanln 21 | ./interactive2.go:64:13: inlining call to fmt.Println 22 | ./interactive2.go:66:13: inlining call to fmt.Println 23 | ./interactive2.go:67:12: inlining call to fmt.Scanln 24 | ./interactive2.go:69:13: inlining call to fmt.Println 25 | ./interactive2.go:71:13: inlining call to fmt.Println 26 | ./interactive2.go:72:12: inlining call to fmt.Scanln 27 | ./interactive2.go:74:13: inlining call to fmt.Println 28 | ./interactive2.go:76:13: inlining call to fmt.Println 29 | ./interactive2.go:77:12: inlining call to fmt.Scanln 30 | ./interactive2.go:79:13: inlining call to fmt.Println 31 | ./interactive2.go:80:19: inlining call to os.(*File).Close 32 | ./interactive2.go:84:13: inlining call to fmt.Println 33 | ./interactive2.go:85:12: inlining call to fmt.Scanln 34 | ./interactive2.go:90:13: inlining call to fmt.Println 35 | ./interactive2.go:91:12: inlining call to fmt.Scanln 36 | ./interactive2.go:97:19: inlining call to os.Open 37 | ./interactive2.go:141:13: inlining call to fmt.Println 38 | ./interactive2.go:142:13: inlining call to fmt.Println 39 | ./interactive2.go:143:13: inlining call to fmt.Println 40 | ./interactive2.go:147:13: inlining call to fmt.Println 41 | ./interactive2.go:148:19: inlining call to os.(*File).Close 42 | ./interactive2.go:11:14: "PID" escapes to heap 43 | ./interactive2.go:11:30: int(~R0) escapes to heap 44 | ./interactive2.go:11:13: []interface {}{...} does not escape 45 | ./interactive2.go:18:12: ... argument does not escape 46 | ./interactive2.go:21:11: make([]byte, 600 * 1024 * 1024) escapes to heap 47 | ./interactive2.go:24:12: ... argument does not escape 48 | ./interactive2.go:27:12: ... argument does not escape 49 | ./interactive2.go:27:13: "Read unexpected amount of bytes" escapes to heap 50 | ./interactive2.go:27:13: n escapes to heap 51 | ./interactive2.go:61:14: "1" escapes to heap 52 | ./interactive2.go:61:13: []interface {}{...} does not escape 53 | ./interactive2.go:64:14: "Reading 5000 index" escapes to heap 54 | ./interactive2.go:64:37: b[5000] escapes to heap 55 | ./interactive2.go:64:13: []interface {}{...} does not escape 56 | ./interactive2.go:66:14: "2" escapes to heap 57 | ./interactive2.go:66:13: []interface {}{...} does not escape 58 | ./interactive2.go:69:14: "Reading 100 000 index" escapes to heap 59 | ./interactive2.go:69:40: b[100000] escapes to heap 60 | ./interactive2.go:69:13: []interface {}{...} does not escape 61 | ./interactive2.go:71:14: "3" escapes to heap 62 | ./interactive2.go:71:13: []interface {}{...} does not escape 63 | ./interactive2.go:74:14: "Reading 104 000 index" escapes to heap 64 | ./interactive2.go:74:40: b[104000] escapes to heap 65 | ./interactive2.go:74:13: []interface {}{...} does not escape 66 | ./interactive2.go:76:14: "4" escapes to heap 67 | ./interactive2.go:76:13: []interface {}{...} does not escape 68 | ./interactive2.go:79:14: "Close file" escapes to heap 69 | ./interactive2.go:79:13: []interface {}{...} does not escape 70 | ./interactive2.go:81:12: ... argument does not escape 71 | ./interactive2.go:84:14: "Force of memory clear" escapes to heap 72 | ./interactive2.go:84:13: []interface {}{...} does not escape 73 | ./interactive2.go:90:14: "Finish" escapes to heap 74 | ./interactive2.go:90:13: []interface {}{...} does not escape 75 | ./interactive2.go:99:12: ... argument does not escape 76 | ./interactive2.go:102:11: make([]byte, 600 * 1024 * 1024) escapes to heap 77 | ./interactive2.go:108:20: ... argument does not escape 78 | ./interactive2.go:108:21: n escapes to heap 79 | ./interactive2.go:141:14: "Reading 5000 index" escapes to heap 80 | ./interactive2.go:141:37: b[5000] escapes to heap 81 | ./interactive2.go:141:13: []interface {}{...} does not escape 82 | ./interactive2.go:142:14: "Reading 100 000 index" escapes to heap 83 | ./interactive2.go:142:40: b[100000] escapes to heap 84 | ./interactive2.go:142:13: []interface {}{...} does not escape 85 | ./interactive2.go:143:14: "Reading 104 000 index" escapes to heap 86 | ./interactive2.go:143:40: b[104000] escapes to heap 87 | ./interactive2.go:143:13: []interface {}{...} does not escape 88 | ./interactive2.go:147:14: "Close file" escapes to heap 89 | ./interactive2.go:147:13: []interface {}{...} does not escape 90 | ./interactive2.go:149:12: ... argument does not escape 91 | :1: leaking param content: .this 92 | */ 93 | 94 | // Buffering 600MB of file in memory. 95 | // Read more in "Efficient Go"; Example 5-2. 96 | 97 | func runOpen() { 98 | fmt.Println("PID", os.Getpid()) 99 | 100 | // TODO(bwplotka): Create big file here, so we can play with it - there is no need to upload so big file to GitHub. 101 | 102 | // Open 686MB file and read 600 MB from it. 103 | f, err := os.Open("test686mbfile.out") 104 | if err != nil { 105 | log.Fatal(err) 106 | } 107 | 108 | b := make([]byte, 600*1024*1024) 109 | _, err = f.Read(b) 110 | if err != nil { 111 | log.Fatal(err) 112 | } 113 | 114 | // Check out: 115 | // ps -ax --format=pid,rss,vsz | grep 116 | // ls -l /proc//map_files 117 | // cat /proc//smaps | grep -A22 c000200000-c025800000 | grep Rss 118 | 119 | /* 120 | c000200000-c025800000 rw-p 00000000 00:00 0 <-- always address of heap 121 | Size: 612352 kB 122 | KernelPageSize: 4 kB 123 | MMUPageSize: 4 kB 124 | Rss: 612352 kB 125 | Pss: 612352 kB 126 | Shared_Clean: 0 kB 127 | Shared_Dirty: 0 kB 128 | Private_Clean: 0 kB 129 | Private_Dirty: 612352 kB 130 | Referenced: 560216 kB 131 | Anonymous: 612352 kB 132 | LazyFree: 0 kB 133 | AnonHugePages: 352256 kB 134 | ShmemPmdMapped: 0 kB 135 | FilePmdMapped: 0 kB 136 | Shared_Hugetlb: 0 kB 137 | Private_Hugetlb: 0 kB 138 | Swap: 0 kB 139 | SwapPss: 0 kB 140 | Locked: 0 kB 141 | THPeligible: 1 142 | VmFlags: rd wr mr mw me ac sd hg 143 | 144 | */ 145 | 146 | fmt.Println("1") 147 | fmt.Scanln() // wait for Enter Key 148 | 149 | fmt.Println("Reading 5000 index", b[5000]) 150 | 151 | fmt.Println("2") 152 | fmt.Scanln() // wait for Enter Key 153 | 154 | fmt.Println("Reading 100 000 index", b[100000]) 155 | 156 | fmt.Println("3") 157 | fmt.Scanln() // wait for Enter Key 158 | 159 | fmt.Println("Reading 104 000 index", b[104000]) 160 | 161 | fmt.Println("4") 162 | fmt.Scanln() // wait for Enter Key 163 | 164 | fmt.Println("Close file") 165 | if err := f.Close(); err != nil { 166 | log.Fatal(err) 167 | } 168 | 169 | fmt.Println("Force of memory clear") 170 | fmt.Scanln() // wait for Enter Key 171 | 172 | b = b[0:] 173 | runtime.GC() 174 | 175 | fmt.Println("Finish") 176 | fmt.Scanln() // wait for Enter Key 177 | 178 | } 179 | 180 | func bookExample2() error { 181 | // Open 686MB file and read 600 MB from it. 182 | f, err := os.Open("test686mbfile.out") 183 | if err != nil { 184 | log.Fatal(err) 185 | } 186 | 187 | b := make([]byte, 600*1024*1024) 188 | n, err := f.Read(b) 189 | if err != nil { 190 | return err 191 | } 192 | if n != len(b) { 193 | return fmt.Errorf("Read unexpected amount of bytes %v", n) 194 | } 195 | 196 | // Check out: 197 | // export PID=642103 && ps -ax --format=pid,rss,vsz | grep $PID && cat /proc/$PID/smaps | grep -A22 c000200000-c025800000 | grep Rss 198 | // cat /proc//smaps | grep -A22 c000200000-c025800000 | grep Rss 199 | // If we would pause the program now `cat /proc//smaps | grep -A22 c000200000-c025800000 | grep Rss` shows already around 600 MB. 200 | /* 201 | c000200000-c025800000 rw-p 00000000 00:00 0 <-- always address of heap 202 | Size: 612352 kB 203 | KernelPageSize: 4 kB 204 | MMUPageSize: 4 kB 205 | Rss: 612352 kB 206 | Pss: 612352 kB 207 | Shared_Clean: 0 kB 208 | Shared_Dirty: 0 kB 209 | Private_Clean: 0 kB 210 | Private_Dirty: 612352 kB 211 | Referenced: 560216 kB 212 | Anonymous: 612352 kB 213 | LazyFree: 0 kB 214 | AnonHugePages: 352256 kB 215 | ShmemPmdMapped: 0 kB 216 | FilePmdMapped: 0 kB 217 | Shared_Hugetlb: 0 kB 218 | Private_Hugetlb: 0 kB 219 | Swap: 0 kB 220 | SwapPss: 0 kB 221 | Locked: 0 kB 222 | THPeligible: 1 223 | VmFlags: rd wr mr mw me ac sd hg 224 | 225 | */ 226 | 227 | fmt.Println("Reading 5000 index", b[5000]) 228 | fmt.Println("Reading 100 000 index", b[100000]) 229 | fmt.Println("Reading 104 000 index", b[104000]) 230 | 231 | // If we would pause the program in each of those steps `cat /proc//smaps | grep -A22 c000200000-c025800000 | grep Rss` shows same around 600 MB. 232 | 233 | fmt.Println("Close file") 234 | if err := f.Close(); err != nil { 235 | log.Fatal(err) 236 | } 237 | 238 | // If we would pause the program now `cat /proc//smaps | grep -A22 c000200000-c025800000 | grep Rss` shows STILL same around 600 MB. 239 | 240 | b = b[0:] 241 | runtime.GC() 242 | 243 | // If we would pause the program now `cat /proc//smaps | grep -A22 c000200000-c025800000 | grep Rss` shows around 140 MB (depends). 244 | return nil 245 | } 246 | -------------------------------------------------------------------------------- /pkg/sum/labeler/labeler_e2e_cmp_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Efficient Go Authors 2 | // Licensed under the Apache License 2.0. 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "strconv" 10 | "testing" 11 | 12 | "github.com/efficientgo/core/testutil" 13 | "github.com/efficientgo/e2e" 14 | e2edb "github.com/efficientgo/e2e/db" 15 | e2einteractive "github.com/efficientgo/e2e/interactive" 16 | e2emonitoring "github.com/efficientgo/e2e/monitoring" 17 | "github.com/thanos-io/objstore/client" 18 | "github.com/thanos-io/objstore/providers/s3" 19 | ) 20 | 21 | func TestLabeler_LabelObject_LargeFiles(t *testing.T) { 22 | t.Skip("Comment this line if you want to run it - it's interactive test. Won't be useful in CI") 23 | 24 | e, err := e2e.NewDockerEnvironment("labeler") 25 | testutil.Ok(t, err) 26 | t.Cleanup(e.Close) 27 | 28 | // Start monitoring. 29 | mon, err := e2emonitoring.Start(e) 30 | testutil.Ok(t, err) 31 | testutil.Ok(t, mon.OpenUserInterfaceInBrowser(`/graph?g0.expr=go_memstats_alloc_bytes%7Bjob%3D~"labelObject.*"%7D&g0.tab=0&g0.stacked=0&g0.show_exemplars=0&g0.range_input=10m&g1.expr=rate(http_request_duration_seconds_sum%5B30s%5D)%20%2F%20rate(http_request_duration_seconds_count%5B30s%5D)&g1.tab=0&g1.stacked=0&g1.show_exemplars=0&g1.range_input=1h`)) 32 | 33 | // Start storage. 34 | minio := e2edb.NewMinio(e, "object-storage", bktName) 35 | testutil.Ok(t, e2e.StartAndWaitReady(minio)) 36 | 37 | // Add test file. 38 | testutil.Ok(t, uploadTestInput(minio, "object.10M.txt", 1e7)) 39 | testutil.Ok(t, uploadTestInput(minio, "object.100M.txt", 1e8)) 40 | 41 | labelers := map[string]e2e.Runnable{labelObject1: nil, labelObject2: nil, labelObject3: nil, labelObject4: nil} 42 | for labelerFunc := range labelers { 43 | // Run program we want to test and benchmark. 44 | labelers[labelerFunc] = e2e.NewInstrumentedRunnable(e, labelerFunc). 45 | WithPorts(map[string]int{"http": 8080}, "http"). 46 | Init(e2e.StartOptions{ 47 | Image: "labeler:test", 48 | LimitCPUs: 4.0, 49 | //EnvVars: map[string]string{ 50 | // // With 2 threads asking max 10 MB, I should not need more, so GC heavily. 51 | // "GOGC": "off", 52 | // "GOMEMLIMIT": "20MiB", 53 | //}, 54 | Command: e2e.NewCommand( 55 | "/labeler", 56 | "-listen-address=:8080", 57 | "-objstore.config="+marshal(t, client.BucketConfig{ 58 | Type: client.S3, 59 | Config: s3.Config{ 60 | Bucket: bktName, 61 | AccessKey: e2edb.MinioAccessKey, 62 | SecretKey: e2edb.MinioSecretKey, 63 | Endpoint: minio.InternalEndpoint(e2edb.AccessPortName), 64 | Insecure: true, 65 | }, 66 | }), 67 | "-function="+labelerFunc, 68 | ), 69 | }) 70 | } 71 | 72 | // Start continuous profiling. 73 | parca := e2e.NewInstrumentedRunnable(e, "parca"). 74 | WithPorts(map[string]int{"http": 7070}, "http"). 75 | Init(e2e.StartOptions{ 76 | Image: "ghcr.io/parca-dev/parca:main-4e20a666", 77 | Command: e2e.NewCommand("/bin/sh", "-c", 78 | `cat << EOF > /shared/data/config.yml && \ 79 | /parca --config-path=/shared/data/config.yml 80 | object_storage: 81 | bucket: 82 | type: "FILESYSTEM" 83 | config: 84 | directory: "./data" 85 | scrape_configs: 86 | - job_name: "labeler" 87 | scrape_interval: "15s" 88 | static_configs: 89 | - targets: 90 | - '`+labelers[labelObject1].InternalEndpoint("http")+`' 91 | - '`+labelers[labelObject2].InternalEndpoint("http")+`' 92 | - '`+labelers[labelObject3].InternalEndpoint("http")+`' 93 | - '`+labelers[labelObject4].InternalEndpoint("http")+`' 94 | profiling_config: 95 | pprof_config: 96 | fgprof: 97 | enabled: true 98 | path: /debug/fgprof/profile 99 | delta: true 100 | EOF 101 | `), 102 | User: strconv.Itoa(os.Getuid()), 103 | Readiness: e2e.NewTCPReadinessProbe("http"), 104 | }) 105 | testutil.Ok(t, e2e.StartAndWaitReady(parca)) 106 | testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+parca.Endpoint("http"))) 107 | 108 | // Load test labeler from 1 clients with k6 and export result to Prometheus. 109 | k6 := e.Runnable("k6").Init(e2e.StartOptions{ 110 | Command: e2e.NewCommandRunUntilStop(), 111 | Image: "grafana/k6:0.39.0", 112 | }) 113 | testutil.Ok(t, e2e.StartAndWaitReady(k6)) 114 | 115 | for _, labelerFunc := range []string{labelObject1, labelObject2, labelObject3, labelObject4} { 116 | l := labelers[labelerFunc] 117 | 118 | testutil.Ok(t, e2e.StartAndWaitReady(l)) 119 | 120 | // 0.5 MB per op alloc. 121 | url10M := fmt.Sprintf("http://%s/label_object?object_id=object.10M.txt", l.InternalEndpoint("http")) 122 | // 5.6MB per op alloc. 123 | url100M := fmt.Sprintf("http://%s/label_object?object_id=object.100M.txt", l.InternalEndpoint("http")) 124 | 125 | testutil.Ok(t, k6.Exec(e2e.NewCommand( 126 | "/bin/sh", "-c", 127 | `cat << EOF | k6 run -u 2 -d 5m - 128 | import http from 'k6/http'; 129 | import { check, sleep } from 'k6'; 130 | 131 | export default function () { 132 | const res = http.get('`+url10M+`'); 133 | let passed = check(res, { 134 | 'is status 200': (r) => r.status === 200, 135 | 'response': (r) => 136 | r.body.includes('{"object_id":"object.10M.txt","sum":31108000000,"checksum":null'), 137 | }); 138 | 139 | const res2 = http.get('`+url100M+`'); 140 | check(res2, { 141 | 'is status 200': (r) => r.status === 200, 142 | 'response': (r) => 143 | r.body.includes('{"object_id":"object.100M.txt","sum":311080000000,"checksum":null'), 144 | }); 145 | } 146 | EOF`))) 147 | testutil.Ok(t, l.Stop()) 148 | } 149 | // Once done, wait for user input so user can explore the results in Prometheus UI and logs. 150 | testutil.Ok(t, e2einteractive.RunUntilEndpointHit()) 151 | } 152 | 153 | func TestLabeler_LabelObject_SmallFiles(t *testing.T) { 154 | t.Skip("Comment this line if you want to run it - it's interactive test. Won't be useful in CI") 155 | 156 | e, err := e2e.NewDockerEnvironment("labeler") 157 | testutil.Ok(t, err) 158 | t.Cleanup(e.Close) 159 | 160 | // Start monitoring. 161 | mon, err := e2emonitoring.Start(e) 162 | testutil.Ok(t, err) 163 | testutil.Ok(t, mon.OpenUserInterfaceInBrowser(`/graph?g0.expr=go_memstats_alloc_bytes%7Bjob%3D~"labelObject.*"%7D&g0.tab=0&g0.stacked=0&g0.show_exemplars=0&g0.range_input=10m&g1.expr=rate(http_request_duration_seconds_sum%5B30s%5D)%20%2F%20rate(http_request_duration_seconds_count%5B30s%5D)&g1.tab=0&g1.stacked=0&g1.show_exemplars=0&g1.range_input=1h`)) 164 | 165 | // Start storage. 166 | minio := e2edb.NewMinio(e, "object-storage", bktName) 167 | testutil.Ok(t, e2e.StartAndWaitReady(minio)) 168 | 169 | // Add test file. 170 | testutil.Ok(t, uploadTestInput(minio, "object.100.txt", 100)) 171 | testutil.Ok(t, uploadTestInput(minio, "object.1000.txt", 1e3)) 172 | 173 | labelers := map[string]e2e.Runnable{labelObject1: nil, labelObject2: nil, labelObject3: nil, labelObject4: nil} 174 | for labelerFunc := range labelers { 175 | // Run program we want to test and benchmark. 176 | labelers[labelerFunc] = e2e.NewInstrumentedRunnable(e, labelerFunc). 177 | WithPorts(map[string]int{"http": 8080}, "http"). 178 | Init(e2e.StartOptions{ 179 | Image: "labeler:test", 180 | LimitCPUs: 4.0, 181 | Command: e2e.NewCommand( 182 | "/labeler", 183 | "-listen-address=:8080", 184 | "-objstore.config="+marshal(t, client.BucketConfig{ 185 | Type: client.S3, 186 | Config: s3.Config{ 187 | Bucket: bktName, 188 | AccessKey: e2edb.MinioAccessKey, 189 | SecretKey: e2edb.MinioSecretKey, 190 | Endpoint: minio.InternalEndpoint(e2edb.AccessPortName), 191 | Insecure: true, 192 | }, 193 | }), 194 | "-function="+labelerFunc, 195 | ), 196 | }) 197 | } 198 | 199 | // Start continuous profiling. 200 | parca := e2e.NewInstrumentedRunnable(e, "parca"). 201 | WithPorts(map[string]int{"http": 7070}, "http"). 202 | Init(e2e.StartOptions{ 203 | Image: "ghcr.io/parca-dev/parca:main-4e20a666", 204 | Command: e2e.NewCommand("/bin/sh", "-c", 205 | `cat << EOF > /shared/data/config.yml && \ 206 | /parca --config-path=/shared/data/config.yml 207 | object_storage: 208 | bucket: 209 | type: "FILESYSTEM" 210 | config: 211 | directory: "./data" 212 | scrape_configs: 213 | - job_name: "labeler" 214 | scrape_interval: "15s" 215 | static_configs: 216 | - targets: 217 | - '`+labelers[labelObject1].InternalEndpoint("http")+`' 218 | - '`+labelers[labelObject2].InternalEndpoint("http")+`' 219 | - '`+labelers[labelObject3].InternalEndpoint("http")+`' 220 | - '`+labelers[labelObject4].InternalEndpoint("http")+`' 221 | profiling_config: 222 | pprof_config: 223 | fgprof: 224 | enabled: true 225 | path: /debug/fgprof/profile 226 | delta: true 227 | EOF 228 | `), 229 | User: strconv.Itoa(os.Getuid()), 230 | Readiness: e2e.NewTCPReadinessProbe("http"), 231 | }) 232 | testutil.Ok(t, e2e.StartAndWaitReady(parca)) 233 | testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+parca.Endpoint("http"))) 234 | 235 | // Load test labeler from 1 clients with k6 and export result to Prometheus. 236 | k6 := e.Runnable("k6").Init(e2e.StartOptions{ 237 | Command: e2e.NewCommandRunUntilStop(), 238 | Image: "grafana/k6:0.39.0", 239 | }) 240 | testutil.Ok(t, e2e.StartAndWaitReady(k6)) 241 | 242 | for _, labelerFunc := range []string{labelObject1, labelObject2, labelObject3, labelObject4} { 243 | l := labelers[labelerFunc] 244 | 245 | testutil.Ok(t, e2e.StartAndWaitReady(l)) 246 | 247 | url100 := fmt.Sprintf("http://%s/label_object?object_id=object.100.txt", l.InternalEndpoint("http")) 248 | url1000 := fmt.Sprintf("http://%s/label_object?object_id=object.1000.txt", l.InternalEndpoint("http")) 249 | 250 | testutil.Ok(t, k6.Exec(e2e.NewCommand( 251 | "/bin/sh", "-c", 252 | `cat << EOF | k6 run -u 12 -d 5m - 253 | import http from 'k6/http'; 254 | import { check, sleep } from 'k6'; 255 | 256 | export default function () { 257 | const res = http.get('`+url100+`'); 258 | let passed = check(res, { 259 | 'is status 200': (r) => r.status === 200, 260 | 'response': (r) => 261 | r.body.includes('{"object_id":"object.10M.txt","sum":311080,"checksum":null'), 262 | }); 263 | 264 | sleep(0.2) 265 | const res2 = http.get('`+url1000+`'); 266 | check(res2, { 267 | 'is status 200': (r) => r.status === 200, 268 | 'response': (r) => 269 | r.body.includes('{"object_id":"object.100M.txt","sum":3110800,"checksum":null'), 270 | }); 271 | sleep(0.2) 272 | } 273 | EOF`))) 274 | testutil.Ok(t, l.Stop()) 275 | } 276 | // Once done, wait for user input so user can explore the results in Prometheus UI and logs. 277 | testutil.Ok(t, e2einteractive.RunUntilEndpointHit()) 278 | } 279 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /.bingo/copyright.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= 4 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 5 | github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 h1:EBTWhcAX7rNQ80RLwLCpHZBBrJuzallFHnF+yMXo928= 6 | github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= 7 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 8 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 9 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 10 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 11 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/efficientgo/tools v0.0.0-20201228165755-e2b84817bf79 h1:Pi2rMrMQeQ45UAZr8Ple+eSk9WCyglYpkFh0r22E00c= 15 | github.com/efficientgo/tools v0.0.0-20201228165755-e2b84817bf79/go.mod h1:jQUsxCcf91LHRhOnGqrx/yrleJbosynzf29/UlCbzlk= 16 | github.com/efficientgo/tools/copyright v0.0.0-20210829154005-c7bad8450208 h1:x1jRmJJG08bP0va7gIX0M39k4vc8szlJNdsoku+sWac= 17 | github.com/efficientgo/tools/copyright v0.0.0-20210829154005-c7bad8450208/go.mod h1:5J0wuuxLMX06WeEgnpf+SvTCptlR9+RHRNO/WEMAwSw= 18 | github.com/efficientgo/tools/core v0.0.0-20210106193344-1108f4e7d16b h1:yi5z8x/FKDHrqtEFiAsxF5b7Sz2+CJrRwBC2kbyhVcA= 19 | github.com/efficientgo/tools/core v0.0.0-20210106193344-1108f4e7d16b/go.mod h1:RJm2+KCRfMUwgEgRte3obd5uLdVY5YbDZjgSMPY0HSA= 20 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 21 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 22 | github.com/felixge/fgprof v0.9.1/go.mod h1:7/HK6JFtFaARhIljgP2IV8rJLIoHDoOYoUphsnGvqxE= 23 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 24 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 25 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 26 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 27 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 28 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 29 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 30 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 31 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 32 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 33 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 34 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 35 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 36 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 37 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 38 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 39 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 40 | github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 41 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 42 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 43 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 44 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 45 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 46 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 47 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 48 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 49 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 50 | github.com/protoconfig/protoconfig/go v0.0.0-20210106192113-733758adefac h1:PWrv6uwNBua14NbS74ukVgXgdRDQPx/2B+Rf6KXXoQk= 51 | github.com/protoconfig/protoconfig/go v0.0.0-20210106192113-733758adefac/go.mod h1:ig8lL2CeTS14ijDIIRoi6pTap0BHc0Xrnke+SKmn9QM= 52 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 53 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 54 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 55 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 56 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= 57 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 58 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 59 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 60 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 61 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 62 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 63 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 64 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 65 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 66 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 67 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 68 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 69 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 70 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 71 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 72 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 73 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 74 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 75 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 76 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 77 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 78 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 79 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 80 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 81 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 82 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 83 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 85 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 86 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 87 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 88 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 89 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 90 | golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 91 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 92 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 93 | golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 94 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 95 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 96 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 97 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 98 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 99 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 100 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 101 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 102 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 103 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 104 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 105 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 106 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 107 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 108 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 109 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 110 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 111 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 112 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 113 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 114 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 115 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 116 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 117 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 118 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 119 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 120 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 121 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 122 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 123 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 124 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 125 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 126 | --------------------------------------------------------------------------------